diff --git a/iib/web/models.py b/iib/web/models.py index e07d9012..c521583b 100644 --- a/iib/web/models.py +++ b/iib/web/models.py @@ -251,7 +251,11 @@ def get_or_create(cls, pull_specification: str) -> Image: :rtype: Image :raise ValidationError: if pull_specification for the image is invalid """ - if '@' not in pull_specification and ':' not in pull_specification: + if ( + '@' not in pull_specification + and ':' not in pull_specification + and pull_specification != "scratch" + ): raise ValidationError( f'Image {pull_specification} should have a tag or a digest specified.' ) diff --git a/iib/workers/tasks/build.py b/iib/workers/tasks/build.py index 99f590d5..cae00a77 100644 --- a/iib/workers/tasks/build.py +++ b/iib/workers/tasks/build.py @@ -807,6 +807,15 @@ def handle_add_request( ), ) from_index_resolved = prebuild_info['from_index_resolved'] + + # FIXME: To be removed when the binaryless image build support is implemented + is_binaryless = prebuild_info['binary_image'] == "scratch" + if is_binaryless: + _cleanup() + log.warning("IIB is not yet able to process binaryless images.") + set_request_state(request_id, 'failed', 'IIB is not yet able to process binaryless images') + return + Opm.set_opm_version(from_index_resolved) with set_registry_token(overwrite_from_index_token, from_index_resolved): is_fbc = is_image_fbc(from_index_resolved) if from_index else False @@ -1061,6 +1070,14 @@ def handle_rm_request( _update_index_image_build_state(request_id, prebuild_info) from_index_resolved = prebuild_info['from_index_resolved'] + # FIXME: To be removed when the binaryless image build support is implemented + is_binaryless = prebuild_info['binary_image'] == "scratch" + if is_binaryless: + _cleanup() + log.warning("IIB is not yet able to process binaryless images.") + set_request_state(request_id, 'failed', 'IIB is not yet able to process binaryless images') + return + Opm.set_opm_version(from_index_resolved) with tempfile.TemporaryDirectory(prefix=f'iib-{request_id}-') as temp_dir: diff --git a/iib/workers/tasks/build_create_empty_index.py b/iib/workers/tasks/build_create_empty_index.py index 21f0601d..03cf4b05 100644 --- a/iib/workers/tasks/build_create_empty_index.py +++ b/iib/workers/tasks/build_create_empty_index.py @@ -95,6 +95,13 @@ def handle_create_empty_index_request( ) from_index_resolved = prebuild_info['from_index_resolved'] prebuild_info['labels'] = labels + # FIXME: To be removed when the binaryless image build support is implemented + is_binaryless = prebuild_info['binary_image'] == "scratch" + if is_binaryless: + _cleanup() + log.warning("IIB is not yet able to process binaryless images.") + set_request_state(request_id, 'failed', 'IIB is not yet able to process binaryless images') + return Opm.set_opm_version(from_index_resolved) if not output_fbc and is_image_fbc(from_index_resolved): diff --git a/iib/workers/tasks/build_fbc_operations.py b/iib/workers/tasks/build_fbc_operations.py index 7029e137..bf4839ba 100644 --- a/iib/workers/tasks/build_fbc_operations.py +++ b/iib/workers/tasks/build_fbc_operations.py @@ -81,6 +81,14 @@ def handle_fbc_operation_request( from_index_resolved = prebuild_info['from_index_resolved'] binary_image_resolved = prebuild_info['binary_image_resolved'] + # FIXME: To be removed when the binaryless image build support is implemented + is_binaryless = prebuild_info['binary_image'] == "scratch" + if is_binaryless: + _cleanup() + log.warning("IIB is not yet able to process binaryless images.") + set_request_state(request_id, 'failed', 'IIB is not yet able to process binaryless images') + return + Opm.set_opm_version(from_index_resolved) prebuild_info['fbc_fragment_resolved'] = resolved_fbc_fragment diff --git a/iib/workers/tasks/build_merge_index_image.py b/iib/workers/tasks/build_merge_index_image.py index 6e0c2cf3..f64017df 100644 --- a/iib/workers/tasks/build_merge_index_image.py +++ b/iib/workers/tasks/build_merge_index_image.py @@ -252,6 +252,14 @@ def handle_merge_request( ) source_from_index_resolved = prebuild_info['source_from_index_resolved'] target_index_resolved = prebuild_info['target_index_resolved'] + # FIXME: To be removed when the binaryless image build support is implemented + is_binaryless = prebuild_info['binary_image'] == "scratch" + if is_binaryless: + _cleanup() + log.warning("IIB is not yet able to process binaryless images.") + set_request_state(request_id, 'failed', 'IIB is not yet able to process binaryless images') + return + Opm.set_opm_version(target_index_resolved) _update_index_image_build_state(request_id, prebuild_info) diff --git a/iib/workers/tasks/utils.py b/iib/workers/tasks/utils.py index fa62e95f..dddde80e 100644 --- a/iib/workers/tasks/utils.py +++ b/iib/workers/tasks/utils.py @@ -137,8 +137,8 @@ def add_max_ocp_version_property(resolved_bundles: List[str], temp_dir: str) -> def get_binary_image_from_config( ocp_version: str, distribution_scope: str, - binary_image_config: Dict[str, Dict[str, str]] = {}, -) -> str: + binary_image_config: Dict[str, Dict[str, str]], +) -> Optional[str]: """ Determine the binary image to be used to build the index image. @@ -146,18 +146,11 @@ def get_binary_image_from_config( :param str distribution_scope: the distribution_scope label value of the index image. :param dict binary_image_config: the dict of config required to identify the appropriate ``binary_image`` to use. - :return: pull specification of the binary_image to be used for this build. + :return: pull specification of the binary_image to be used for this build when found. :rtype: str - :raises IIBError: when the config value for the ocp_version and distribution_scope is missing. """ + binary_image_config = binary_image_config or {} binary_image = binary_image_config.get(distribution_scope, {}).get(ocp_version, None) - if not binary_image: - raise IIBError( - 'IIB does not have a configured binary_image for' - f' distribution_scope : {distribution_scope} and ocp_version: {ocp_version}.' - ' Please specify a binary_image value in the request.' - ) - return binary_image @@ -270,11 +263,22 @@ def __repr__(self) -> str: def binary_image(self, index_info: IndexImageInfo, distribution_scope: str) -> str: """Get binary image based on self configuration, index image info and distribution scope.""" + ocp_version = index_info['ocp_version'] + config_binary_image = get_binary_image_from_config( + ocp_version, distribution_scope, self.binary_image_config + ) if not self._binary_image: - binary_image_ocp_version = index_info['ocp_version'] - return get_binary_image_from_config( - binary_image_ocp_version, distribution_scope, self.binary_image_config - ) + if not config_binary_image: + raise IIBError( + 'IIB does not have a configured binary_image for' + f' distribution_scope : {distribution_scope} and ocp_version: {ocp_version}.' + ' Please specify a binary_image value in the request.' + ) + return config_binary_image + if self._binary_image == "scratch" and config_binary_image != "scratch": + raise IIBError(f"The index image should not be binaryless for {ocp_version}.") + elif self._binary_image != "scratch" and config_binary_image == "scratch": + raise IIBError(f"The index image must be built as a binaryless for {ocp_version}.") return self._binary_image @@ -1192,9 +1196,12 @@ def prepare_request_for_build( ) binary_image = build_request_config.binary_image(request_index, distribution_scope) - - binary_image_resolved = get_resolved_image(binary_image) - binary_image_arches = get_image_arches(binary_image_resolved) + if binary_image == "scratch": # binaryless image mode + binary_image_resolved = binary_image + binary_image_arches = arches + else: + binary_image_resolved = get_resolved_image(binary_image) + binary_image_arches = get_image_arches(binary_image_resolved) if not arches.issubset(binary_image_arches): raise IIBError( diff --git a/tests/test_web/test_api_v1.py b/tests/test_web/test_api_v1.py index 7771ef3d..f425539d 100644 --- a/tests/test_web/test_api_v1.py +++ b/tests/test_web/test_api_v1.py @@ -814,6 +814,7 @@ def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env, @pytest.mark.parametrize( ( + 'binary_image', 'overwrite_from_index', 'overwrite_from_index_token', 'bundles', @@ -822,11 +823,11 @@ def test_add_bundle_from_index_and_add_arches_missing(mock_smfsc, db, auth_env, 'graph_update_mode', ), ( - (False, None, ['some:thing'], None, None, None), - (False, None, ['some:thing'], 'some:thing', None, 'semver'), - (False, None, [], 'some:thing', 'Prod', 'semver-skippatch'), - (True, 'username:password', ['some:thing'], 'some:thing', 'StagE', 'replaces'), - (True, 'username:password', [], 'some:thing', 'DeV', 'semver'), + ('binary:image', False, None, ['some:thing'], None, None, None), + ('binary:image', False, None, ['some:thing'], 'some:thing', None, 'semver'), + ('binary:image', False, None, [], 'some:thing', 'Prod', 'semver-skippatch'), + ('scratch', True, 'username:password', ['some:thing'], 'some:thing', 'StagE', 'replaces'), + ('scratch', True, 'username:password', [], 'some:thing', 'DeV', 'semver'), ), ) @mock.patch('iib.web.api_v1.handle_add_request') @@ -844,10 +845,11 @@ def test_add_bundle_success( from_index, distribution_scope, graph_update_mode, + binary_image, ): app.config['IIB_GRAPH_MODE_INDEX_ALLOW_LIST'] = [from_index] data = { - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'add_arches': ['s390x'], 'organization': 'org', 'cnr_token': 'token', @@ -872,7 +874,7 @@ def test_add_bundle_success( 'arches': [], 'batch': 1, 'batch_annotations': None, - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'binary_image_resolved': None, 'build_tags': [], 'bundle_mapping': {}, @@ -1414,12 +1416,13 @@ def test_patch_request_regenerate_bundle_success( mock_smfsc.assert_called_once_with(mock.ANY) +@pytest.mark.parametrize("binary_image", ('binary:image', 'scratch')) @mock.patch('iib.web.api_v1.handle_rm_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') -def test_remove_operator_success(mock_smfsc, mock_rm, db, auth_env, client): +def test_remove_operator_success(mock_smfsc, mock_rm, binary_image, db, auth_env, client): data = { 'operators': ['some:thing'], - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'from_index': 'index:image', } @@ -1427,7 +1430,7 @@ def test_remove_operator_success(mock_smfsc, mock_rm, db, auth_env, client): 'arches': [], 'batch': 1, 'batch_annotations': None, - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'binary_image_resolved': None, 'bundle_mapping': {}, 'distribution_scope': None, @@ -1960,16 +1963,17 @@ def test_regenerate_add_rm_batch_invalid_input(payload, error_msg, app, auth_env assert rv.json == {'error': error_msg} +@pytest.mark.parametrize("binary_image", ('binary:image', 'scratch')) @pytest.mark.parametrize('distribution_scope', (None, 'stage')) @mock.patch('iib.web.api_v1.handle_merge_request') @mock.patch('iib.web.api_v1.messaging.send_message_for_state_change') def test_merge_index_image_success( - mock_smfsc, mock_merge, app, db, auth_env, client, distribution_scope + mock_smfsc, mock_merge, binary_image, app, db, auth_env, client, distribution_scope ): app.config['IIB_GRAPH_MODE_INDEX_ALLOW_LIST'] = ['target_index:image'] data = { 'deprecation_list': ['some@sha256:bundle'], - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'source_from_index': 'source_index:image', 'target_index': 'target_index:image', 'build_tags': [], @@ -1983,7 +1987,7 @@ def test_merge_index_image_success( 'arches': [], 'batch': 1, 'batch_annotations': None, - 'binary_image': 'binary:image', + 'binary_image': binary_image, 'binary_image_resolved': None, 'build_tags': [], 'deprecation_list': ['some@sha256:bundle'], @@ -2199,7 +2203,9 @@ def test_merge_index_image_fail_on_invalid_params( ( ('some:thing', 'from:variable', None), ('some:thing', 'binary:image', {'version': 'v4.6'}), + ('some:thing', 'scratch', {'version': 'v4.6'}), ('some:thing', 'binary:image', None), + ('some:thing', 'scratch', None), ), ) @mock.patch('iib.web.api_v1.handle_create_empty_index_request.apply_async') diff --git a/tests/test_workers/test_tasks/test_utils.py b/tests/test_workers/test_tasks/test_utils.py index 4f231fd2..4c1f0210 100644 --- a/tests/test_workers/test_tasks/test_utils.py +++ b/tests/test_workers/test_tasks/test_utils.py @@ -1007,6 +1007,186 @@ def test_prepare_request_for_build_merge_index_img(mock_gia, mock_gri, mock_giii } +@pytest.mark.parametrize( + 'from_index, from_index_arches, bundles, binary_image,' + 'expected_bundle_mapping, distribution_scope, resolved_distribution_scope, binary_image_config', + ( + ( + 'some-index:latest', + {'amd64'}, + None, + 'scratch', + {}, + None, + 'prod', + {'prod': {'v4.6': 'scratch'}}, + ), + (None, set(), None, 'scratch', {}, None, 'prod', {'prod': {'v4.5': 'scratch'}}), + ( + 'some-index:latest', + {'amd64'}, + None, + '', + {}, + None, + 'prod', + {'prod': {'v4.6': 'scratch'}}, + ), + ( + None, + set(), + ['quay.io/some-bundle:v1', 'quay.io/some-bundle2:v1'], + None, + { + 'some-bundle': ['quay.io/some-bundle:v1'], + 'some-bundle2': ['quay.io/some-bundle2:v1'], + }, + None, + 'prod', + {'prod': {'v4.5': 'scratch'}}, + ), + ( + 'some-index:latest', + set(), + ['quay.io/some-bundle:v1', 'quay.io/some-bundle2:v1'], + '', + { + 'some-bundle': ['quay.io/some-bundle:v1'], + 'some-bundle2': ['quay.io/some-bundle2:v1'], + }, + None, + 'prod', + {'prod': {'v4.6': 'scratch'}}, + ), + ), +) +@mock.patch('iib.workers.tasks.build.set_request_state') +@mock.patch('iib.workers.tasks.utils.set_request_state') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.utils.get_image_arches') +@mock.patch('iib.workers.tasks.utils.get_image_label') +@mock.patch('iib.workers.tasks.build.update_request') +def test_prepare_request_for_build_binaryless( + mock_ur, + mock_gil, + mock_gia, + mock_gri, + mock_srs, + mock_srs2, + from_index, + from_index_arches, + bundles, + binary_image, + expected_bundle_mapping, + distribution_scope, + resolved_distribution_scope, + binary_image_config, +) -> None: + binary_image_resolved = "scratch" + from_index_resolved = None + expected_payload_keys = {'binary_image_resolved', 'state', 'state_reason'} + gil_side_effect = [] + ocp_version = 'v4.5' + if expected_bundle_mapping: + expected_payload_keys.add('bundle_mapping') + if from_index: + from_index_name = from_index.split(':', 1)[0] + from_index_resolved = f'{from_index_name}@sha256:bcdefg' + index_resolved = f'{from_index_name}@sha256:abcdef1234' + mock_gri.side_effect = [from_index_resolved, binary_image_resolved, index_resolved] + mock_gia.return_value = from_index_arches + expected_payload_keys.add('from_index_resolved') + gil_side_effect = ['v4.6', resolved_distribution_scope] + ocp_version = 'v4.6' + else: + index_resolved = f'index-image@sha256:abcdef1234' + mock_gri.side_effect = [binary_image_resolved, index_resolved] + mock_gia.return_value = from_index_arches + gil_side_effect = [] + + if bundles: + bundle_side_effects = [bundle.rsplit('/', 1)[1].split(':', 1)[0] for bundle in bundles] + gil_side_effect.extend(bundle_side_effects) + + mock_gil.side_effect = gil_side_effect + + rv = utils.prepare_request_for_build( + 1, + utils.RequestConfigAddRm( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=None, + add_arches=["amd64"], + bundles=bundles, + distribution_scope="prod", + binary_image_config=binary_image_config, + ), + ) + + assert rv == { + 'arches': {'amd64'}, + 'binary_image': "scratch", + 'binary_image_resolved': "scratch", + 'bundle_mapping': expected_bundle_mapping, + 'from_index_resolved': from_index_resolved, + 'ocp_version': ocp_version, + # want to verify that the output is always lower cased. + 'distribution_scope': resolved_distribution_scope.lower(), + 'source_from_index_resolved': None, + 'source_ocp_version': 'v4.5', + 'target_index_resolved': None, + 'target_ocp_version': 'v4.6', + } + + +@pytest.mark.parametrize( + 'from_index, binary_image, binary_image_config,error', + ( + ( + 'some-index:latest', + 'binary-image:latest', + {'prod': {'v4.5': 'scratch'}}, + 'The index image must be built as a binaryless for v4.5', + ), + ( + 'some-index:latest', + 'scratch', + {'prod': {'v4.5': 'binary-image:latest'}}, + 'The index image should not be binaryless for v4.5', + ), + ), +) +@mock.patch('iib.workers.tasks.utils.set_request_state') +@mock.patch('iib.workers.tasks.utils.get_index_image_info') +@mock.patch('iib.workers.tasks.utils.get_resolved_image') +@mock.patch('iib.workers.tasks.utils.get_image_arches') +def test_prepare_request_for_build_binaryless_invalid_conditions( + mock_gia, mock_gri, mock_giii, mock_srs, from_index, binary_image, binary_image_config, error +) -> None: + mock_giii.return_value = { + 'resolved_from_index': None, + 'ocp_version': 'v4.5', + 'arches': set(), + 'resolved_distribution_scope': 'prod', + } + mock_gri.return_value = 'scratch' + mock_gia.return_value = {'amd64'} + + with pytest.raises(IIBError, match=error): + utils.prepare_request_for_build( + 1, + utils.RequestConfigAddRm( + _binary_image=binary_image, + from_index=from_index, + overwrite_from_index_token=None, + add_arches="amd64", + bundles=['foo', 'bar'], + distribution_scope="prod", + binary_image_config=binary_image_config, + ), + ) + + @mock.patch('iib.workers.tasks.utils.set_request_state') @mock.patch('iib.workers.tasks.utils.get_resolved_image') @mock.patch('iib.workers.tasks.utils.get_image_arches') @@ -1062,8 +1242,16 @@ def test_validate_distribution_scope( def test_get_binary_image_config_no_config_val(): + index_info = { + 'resolved_from_index': 'some_other_image@sha256', + 'ocp_version': 'v4.5', + 'arches': {'amd64'}, + 'resolved_distribution_scope': 'prod', + } + b_img_cfg = {'prod': {'v4.6': 'binary_image'}} + rc = utils.RequestConfig(distribution_scope="prod", binary_image_config=b_img_cfg) with pytest.raises(IIBError, match='IIB does not have a configured binary_image.+'): - utils.get_binary_image_from_config('prod', 'v4.5', {'prod': {'v4.6': 'binary_image'}}) + rc.binary_image(index_info, "prod") @pytest.mark.parametrize(