diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 4fd865f..64f3d62 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -3,31 +3,6 @@ name: Continuous testing & Linting on: [push] jobs: - test36: - - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.6] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e .[dev] - - name: Lint with flake8 - run: | - flake8 img_proof - flake8 tests --exclude=data - - name: Test with pytest - run: | - pytest --cov=img_proof - testall: runs-on: ubuntu-latest diff --git a/img_proof/ipa_gce.py b/img_proof/ipa_gce.py index 02e3910..006737b 100644 --- a/img_proof/ipa_gce.py +++ b/img_proof/ipa_gce.py @@ -37,8 +37,8 @@ from google.oauth2 import service_account from google.auth.exceptions import RefreshError from google.auth.transport.requests import AuthorizedSession -from googleapiclient import discovery -from googleapiclient.errors import HttpError +from google.cloud import compute_v1 +from google.api_core.extended_operation import ExtendedOperation def get_message_from_http_error(error, resource_name): @@ -69,15 +69,6 @@ def handle_gce_http_errors(type_name, resource_name): """ try: yield - except HttpError as error: - message = get_message_from_http_error(error, resource_name) - - raise GCECloudException( - 'Unable to retrieve {type_name}: {error}'.format( - type_name=type_name, - error=message - ) - ) from error except Exception as error: raise GCECloudException( 'Unable to retrieve {type_name}: {error}'.format( @@ -141,7 +132,12 @@ def post_init(self): self.sev = None self.credentials = self._get_credentials() - self.compute_driver = self._get_driver() + self.instances_client = compute_v1.InstancesClient( + credentials=self.credentials + ) + self.zone_operations_client = compute_v1.ZoneOperationsClient( + credentials=self.credentials + ) self._validate_region() @@ -198,23 +194,14 @@ def _get_credentials(self): return creds - def _get_driver(self): - """Get authenticated GCE driver.""" - return discovery.build( - 'compute', - 'beta', - credentials=self.credentials, - cache_discovery=False - ) - def _get_instance(self): """Retrieve instance matching instance_id.""" with handle_gce_http_errors('instance', self.running_instance_id): - instance = self.compute_driver.instances().get( + instance = self.instances_client.get( project=self.service_account_project, zone=self.region, instance=self.running_instance_id - ).execute() + ) return instance @@ -232,11 +219,15 @@ def _get_network(self, network_id): If network not found GCE will raise a 404 error. """ + networks_client = compute_v1.NetworksClient( + credentials=self.credentials + ) + with handle_gce_http_errors('network', network_id): - network = self.compute_driver.networks().get( + network = networks_client.get( project=self.service_account_project, network=network_id - ).execute() + ) return network @@ -246,15 +237,19 @@ def _get_subnet(self, subnet_id): If subnet not found GCE will raise a 404 error. """ + subnetworks_client = compute_v1.SubnetworksClient( + credentials=self.credentials + ) + with handle_gce_http_errors('subnet', subnet_id): # Subnet lives in a region whereas self.region # is a specific zone (us-west1-a). region = '-'.join(self.region.split('-')[:-1]) - subnet = self.compute_driver.subnetworks().get( + subnet = subnetworks_client.get( project=self.service_account_project, region=region, subnetwork=subnet_id - ).execute() + ) return subnet @@ -264,12 +259,16 @@ def _get_instance_type(self, type_name): If type not found GCE will raise a 404 error. """ + machine_types_client = compute_v1.MachineTypesClient( + credentials=self.credentials + ) + with handle_gce_http_errors('instance type', type_name): - machine_type = self.compute_driver.machineTypes().get( + machine_type = machine_types_client.get( project=self.service_account_project, zone=self.region, - machineType=type_name - ).execute() + machine_type=type_name + ) return machine_type @@ -279,11 +278,15 @@ def _get_image(self, image_name): If image is not found GCE will raise a 404 error. """ + images_client = compute_v1.ImagesClient( + credentials=self.credentials + ) + with handle_gce_http_errors('image', image_name): - image = self.compute_driver.images().get( + image = images_client.get( project=self.image_project or self.service_account_project, image=image_name - ).execute() + ) return image @@ -293,12 +296,16 @@ def _get_disk(self, disk_name): If disk is not found GCE will raise a 404 error. """ + disks_client = compute_v1.DisksClient( + credentials=self.credentials + ) + with handle_gce_http_errors('disk', disk_name): - disk = self.compute_driver.disks().get( + disk = disks_client.get( project=self.service_account_project, zone=self.region, disk=disk_name - ).execute() + ) return disk @@ -310,21 +317,21 @@ def _get_network_config(self, subnet_id, use_gvnic=False): network. Otherwise use the default network. """ interface = { - 'accessConfigs': [{ + 'access_configs': [{ 'name': 'External NAT', - 'type': 'ONE_TO_ONE_NAT' + 'type_': 'ONE_TO_ONE_NAT' }] } if use_gvnic: - interface['nicType'] = 'GVNIC' + interface['nic_type'] = 'GVNIC' if subnet_id: subnet = self._get_subnet(subnet_id) - interface['subnetwork'] = subnet['selfLink'] - interface['network'] = subnet['network'] + interface['subnetwork'] = subnet.self_link + interface['network'] = subnet.network.self_link else: - interface['network'] = self._get_network('default')['selfLink'] + interface['network'] = self._get_network('default').self_link return interface @@ -340,9 +347,9 @@ def get_shielded_instance_config( Return with default values unless overridden by args. """ shielded_instance_config = { - 'enableSecureBoot': enable_secure_boot, - 'enableVtpm': enable_vtpm, - 'enableIntegrityMonitoring': enable_integrity_monitoring + 'enable_secure_boot': enable_secure_boot, + 'enable_vtpm': enable_vtpm, + 'enable_integrity_monitoring': enable_integrity_monitoring } return shielded_instance_config @@ -370,59 +377,62 @@ def get_instance_config( 'metadata': { 'items': [ {'key': 'ssh-keys', 'value': ssh_key}, - {'key': 'enable-guest-attributes', 'value': True} + {'key': 'enable-guest-attributes', 'value': 'true'} ] }, - 'serviceAccounts': [{ + 'service_accounts': [{ 'email': service_account_email, 'scopes': [ 'https://www.googleapis.com/auth/devstorage.read_only' ] }], - 'machineType': machine_type, + 'machine_type': machine_type, 'disks': [{ - 'autoDelete': auto_delete, + 'auto_delete': auto_delete, 'boot': boot_disk, - 'type': disk_type, + 'type_': disk_type, 'mode': disk_mode, - 'deviceName': instance_name, - 'initializeParams': { - 'diskName': instance_name, - 'sourceImage': source_image, - 'diskSizeGb': root_disk_size, + 'device_name': instance_name, + 'initialize_params': { + 'disk_name': instance_name, + 'source_image': source_image, + 'disk_size_gb': root_disk_size, 'architecture': architecture } }], - 'networkInterfaces': network_interfaces, + 'network_interfaces': network_interfaces, 'name': instance_name } guest_os_features = [] if shielded_instance_config: - config['shieldedInstanceConfig'] = shielded_instance_config - guest_os_features.append({'type': 'UEFI_COMPATIBLE'}) + config['shielded_instance_config'] = shielded_instance_config + guest_os_features.append({'type_': 'UEFI_COMPATIBLE'}) if sev: - config['confidentialInstanceConfig'] = { - 'confidentialInstanceType': sev, - 'enableConfidentialCompute': True + config['confidential_instance_config'] = { + # 'confidential_instance_type': sev, + 'enable_confidential_compute': True } - config['scheduling'] = {'onHostMaintenance': 'TERMINATE'} + config['scheduling'] = {'on_host_maintenance': 'TERMINATE'} if sev == 'SEV_SNP': - config['minCpuPlatform'] = 'AMD Milan' - guest_os_features.append({'type': 'SEV_SNP_CAPABLE'}) + config['min_cpu_platform'] = 'AMD Milan' + guest_os_features.append({'type_': 'SEV_SNP_CAPABLE'}) else: - guest_os_features.append({'type': 'SEV_CAPABLE'}) + guest_os_features.append({'type_': 'SEV_CAPABLE'}) if use_gvnic: - guest_os_features.append({'type': 'GVNIC'}) + guest_os_features.append({'type_': 'GVNIC'}) if guest_os_features: - config['disks'][0]['guestOsFeatures'] = guest_os_features + config['disks'][0]['guest_os_features'] = guest_os_features - return config + instance = compute_v1.Instance( + mapping=config + ) + return instance def _launch_instance(self): """Launch an instance of the given image.""" @@ -431,8 +441,8 @@ def _launch_instance(self): machine_type = self._get_instance_type( self.instance_type or GCE_DEFAULT_TYPE - )['selfLink'] - source_image = self._get_image(self.image_id)['selfLink'] + ).self_link + source_image = self._get_image(self.image_id).self_link network_interfaces = [ self._get_network_config(self.subnet_id, self.use_gvnic) ] @@ -457,34 +467,12 @@ def _launch_instance(self): ) try: - response = self.compute_driver.instances().insert( + request = compute_v1.InsertInstanceRequest( project=self.service_account_project, zone=self.region, - body=self.get_instance_config(**kwargs) - ).execute() - except HttpError as error: - with suppress(AttributeError): - # In python 3.5 content is bytes - error.content = error.content.decode() - - error_obj = json.loads(error.content)['error'] - - try: - message = error_obj['message'] - except (AttributeError, KeyError): - message = 'Unknown exception.' - - if error_obj['code'] == 412: - # 412 is conditionNotmet - error_class = IpaRetryableError - else: - error_class = GCECloudException - - raise error_class( - 'Failed to launch instance: {message}'.format( - message=message - ) - ) from error + instance_resource=self.get_instance_config(**kwargs) + ) + operation = self.instances_client.insert(request=request) except Exception as error: raise GCECloudException( 'Failed to launch instance: {message}'.format( @@ -492,21 +480,7 @@ def _launch_instance(self): ) ) from error - operation = self._wait_on_operation(response['name']) - - if 'error' in operation and operation['error'].get('errors'): - error = operation['error']['errors'][0] - - if error['code'] in ('QUOTA_EXCEEDED', 'PRECONDITION_FAILED'): - error_class = IpaRetryableError - else: - error_class = GCECloudException - - raise error_class( - 'Failed to launch instance: {message}'.format( - message=error['message'] - ) - ) + self.wait_for_extended_operation(operation, 'instance creation') self._wait_on_instance( 'RUNNING', @@ -517,16 +491,16 @@ def _set_image_id(self): """Set the image_id instance variable based on boot disk.""" instance = self._get_instance() - for disk_info in instance['disks']: - if disk_info.get('boot'): - disk_name = disk_info['source'].rsplit('/', maxsplit=1)[-1] + for disk_info in instance.disks: + if disk_info.boot: + disk_name = disk_info.source.rsplit('/', maxsplit=1)[-1] break disk = self._get_disk(disk_name) # Example sourceImage format: # projects/debian-cloud/global/images/opensuse-leap-15.0-YYYYMMDD - self.image_id = disk['sourceImage'].rsplit('/', maxsplit=1)[-1] + self.image_id = disk.source_image.rsplit('/', maxsplit=1)[-1] def _validate_region(self): """Validate region was passed in and is a valid GCE zone.""" @@ -536,11 +510,15 @@ def _validate_region(self): 'Example: us-west1-a' ) + zones_client = compute_v1.ZonesClient( + credentials=self.credentials + ) + try: - zone = self.compute_driver.zones().get( + zone = zones_client.get( project=self.service_account_project, zone=self.region - ).execute() + ) except Exception: zone = None @@ -555,7 +533,7 @@ def _validate_region(self): def _get_instance_state(self): """Attempt to retrieve the state of the instance.""" instance = self._get_instance() - return instance['status'] + return instance.status def _is_instance_running(self): """Return True if instance is in running state.""" @@ -565,25 +543,25 @@ def _set_instance_ip(self): """Retrieve and set the instance ip address.""" instance = self._get_instance() - interface = instance['networkInterfaces'][0] + interface = instance.network_interfaces[0] try: - self.instance_ip = interface['accessConfigs'][0]['natIP'] - except (KeyError, IndexError): - try: - self.instance_ip = interface['networkIP'] - except KeyError: - raise GCECloudException( - 'IP address for instance: %s cannot be found.' - % self.running_instance_id - ) + self.instance_ip = interface.access_configs[0].nat_i_p + except IndexError: + self.instance_ip = interface.network_i_p + + if not self.instance_ip: + raise GCECloudException( + 'IP address for instance: %s cannot be found.' + % self.running_instance_id + ) def _start_instance(self): """Start the instance.""" - self.compute_driver.instances().start( + self.instances_client.start( project=self.service_account_project, zone=self.region, instance=self.running_instance_id - ).execute() + ) self._wait_on_instance( 'RUNNING', @@ -592,11 +570,11 @@ def _start_instance(self): def _stop_instance(self): """Stop the instance.""" - self.compute_driver.instances().stop( + self.instances_client.stop( project=self.service_account_project, zone=self.region, instance=self.running_instance_id - ).execute() + ) # In GCE an instance that is stopped has a state of TERMINATED: # https://cloud.google.com/compute/docs/instances/instance-life-cycle @@ -607,22 +585,22 @@ def _stop_instance(self): def _terminate_instance(self): """Terminate the instance.""" - self.compute_driver.instances().delete( + self.instances_client.delete( project=self.service_account_project, zone=self.region, instance=self.running_instance_id - ).execute() + ) def get_console_log(self): """ Return console log output if it is available. """ - output = self.compute_driver.instances().getSerialPortOutput( + output = self.instances_client.get_serial_port_output( project=self.service_account_project, zone=self.region, instance=self.running_instance_id - ).execute() - return output.get('contents', '') + ) + return output.contents def _wait_on_operation(self, operation_name, timeout=600, wait_period=10): start = time.time() @@ -631,11 +609,37 @@ def _wait_on_operation(self, operation_name, timeout=600, wait_period=10): while time.time() < end: time.sleep(wait_period) - operation = self.compute_driver.zoneOperations().get( + operation = self.zone_operations_client.get( project=self.service_account_project, zone=self.region, operation=operation_name - ).execute() + ) - if operation['status'] == 'DONE': + if operation.status == 'DONE': return operation + + def wait_for_extended_operation( + self, + operation: ExtendedOperation, + verbose_name: str = 'operation', + timeout: int = 300 + ): + retryable_errors = ('QUOTA_EXCEEDED', 'PRECONDITION_FAILED') + result = operation.result(timeout=timeout) + + if operation.warnings: + for warning in operation.warnings: + self.logger.warning(f'{warning.code}: {warning.message}') + + if operation.error_code: + if operation.error_code in retryable_errors: + error_class = IpaRetryableError + else: + error_class = GCECloudException + + raise error_class( + f'Failed to launch instance: {operation.error_code}: ' + f'{operation.error_message}' + ) + + return result diff --git a/package/python3-img-proof.spec b/package/python-img-proof.spec similarity index 51% rename from package/python3-img-proof.spec rename to package/python-img-proof.spec index 8a05690..0da6666 100644 --- a/package/python3-img-proof.spec +++ b/package/python-img-proof.spec @@ -1,5 +1,5 @@ # -# spec file for package python3-img-proof +# spec file for package python-img-proof # # Copyright (c) 2020 SUSE LLC # @@ -15,9 +15,12 @@ # Please submit bugfixes or comments via https://bugs.opensuse.org/ # - +%define upstream_name img-proof +%define python python +%{?sle15_python_module_pythons} %bcond_without test -Name: python3-img-proof + +Name: python-img-proof Version: 7.24.0 Release: 0 Summary: Command line and API for testing custom images @@ -26,49 +29,51 @@ Group: Development/Languages/Python URL: https://github.com/SUSE-Enceladus/img-proof Source: https://files.pythonhosted.org/packages/source/i/img-proof/img-proof-%{version}.tar.gz BuildRequires: python-rpm-macros -BuildRequires: python3-PyYAML -BuildRequires: python3-aliyun-python-sdk-core -BuildRequires: python3-aliyun-python-sdk-ecs -BuildRequires: python3-msrestazure >= 0.6.0 -BuildRequires: python3-azure-identity -BuildRequires: python3-azure-mgmt-compute -BuildRequires: python3-azure-mgmt-network -BuildRequires: python3-azure-mgmt-resource -BuildRequires: python3-boto3 -BuildRequires: python3-click -BuildRequires: python3-click-man -BuildRequires: python3-devel -BuildRequires: python3-google-api-python-client -BuildRequires: python3-google-auth -BuildRequires: python3-oci-sdk -BuildRequires: python3-paramiko -BuildRequires: python3-pytest -BuildRequires: python3-setuptools -BuildRequires: python3-pytest-testinfra -BuildRequires: python3-pytest-json-report +BuildRequires: %{python_module PyYAML} +BuildRequires: %{python_module aliyun-python-sdk-core} +BuildRequires: %{python_module aliyun-python-sdk-ecs} +BuildRequires: %{python_module msrestazure >= 0.6.0} +BuildRequires: %{python_module azure-identity} +BuildRequires: %{python_module azure-mgmt-compute} +BuildRequires: %{python_module azure-mgmt-network} +BuildRequires: %{python_module azure-mgmt-resource} +BuildRequires: %{python_module boto3} +BuildRequires: %{python_module click} +BuildRequires: %{python_module click-man} +BuildRequires: %{python_module devel} +BuildRequires: %{python_module google-cloud-compute} +BuildRequires: %{python_module google-auth} +BuildRequires: %{python_module oci-sdk} +BuildRequires: %{python_module paramiko} +BuildRequires: %{python_module pytest} +BuildRequires: %{python_module setuptools} +BuildRequires: %{python_module pytest-testinfra} +BuildRequires: %{python_module pytest-json-report} %if %{with test} -BuildRequires: python3-coverage -BuildRequires: python3-pytest-cov +BuildRequires: %{python_module coverage} +BuildRequires: %{python_module pytest-cov} %endif -Requires: python3-PyYAML -Requires: python3-aliyun-python-sdk-core -Requires: python3-aliyun-python-sdk-ecs -Requires: python3-msrestazure >= 0.6.0 -Requires: python3-azure-identity -Requires: python3-azure-mgmt-compute -Requires: python3-azure-mgmt-network -Requires: python3-azure-mgmt-resource -Requires: python3-boto3 -Requires: python3-click -Requires: python3-google-api-python-client -Requires: python3-google-auth -Requires: python3-oci-sdk -Requires: python3-paramiko -Requires: python3-pytest -Requires: python3-pytest-testinfra -Requires: python3-pytest-json-report +Requires: python-PyYAML +Requires: python-aliyun-python-sdk-core +Requires: python-aliyun-python-sdk-ecs +Requires: python-msrestazure >= 0.6.0 +Requires: python-azure-identity +Requires: python-azure-mgmt-compute +Requires: python-azure-mgmt-network +Requires: python-azure-mgmt-resource +Requires: python-boto3 +Requires: python-click +Requires: python-google-cloud-compute +Requires: python-google-auth +Requires: python-oci-sdk +Requires: python-paramiko +Requires: python-pytest +Requires: python-pytest-testinfra +Requires: python-pytest-json-report BuildArch: noarch Obsoletes: python3-ipa < 7.24.0 +Provides: python3-img-proof = %{version} +Obsoletes: python3-img-proof < %{version} %description img-proof provides a command line utility to test images in @@ -77,8 +82,8 @@ the Public Cloud (AWS, Azure, GCE, etc.). %package tests Summary: Infrastructure tests for img-proof Group: Development/Languages/Python -Requires: python3-susepubliccloudinfo -PreReq: python3-img-proof = %{version} +Requires: python-susepubliccloudinfo +PreReq: python-img-proof = %{version} Obsoletes: python3-ipa-tests < 7.24.0 %description tests @@ -88,12 +93,12 @@ Directory of infrastructure tests for testing images. %setup -q -n img-proof-%{version} %build -python3 setup.py build +%pyproject_wheel mkdir -p man/man1 python3 setup.py --command-packages=click_man.commands man_pages --target man/man1 %install -python3 setup.py install --prefix=%{_prefix} --root=%{buildroot} +%pyproject_install install -d -m 755 %{buildroot}/%{_mandir}/man1 install -m 644 man/man1/*.1 %{buildroot}/%{_mandir}/man1 @@ -113,7 +118,7 @@ python3 -m pytest --cov=img_proof --ignore=tests/data --ignore=usr %doc CHANGES.md CONTRIBUTING.md README.md %{_mandir}/man1/* %{_bindir}/img-proof -%{python3_sitelib}/* +%{python_sitelib}/* %files tests %defattr(-,root,root) diff --git a/requirements.txt b/requirements.txt index aa8bc91..d7af6aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ PyYAML pytest-testinfra oci google-auth -google-api-python-client +google-cloud-compute aliyun-python-sdk-core aliyun-python-sdk-ecs pytest-json-report diff --git a/setup.py b/setup.py index feaf8f9..8f9d736 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ ] }, include_package_data=True, - python_requires='>=3.4', + python_requires='>=3.7', install_requires=requirements, extras_require={ 'dev': dev_requirements, @@ -79,8 +79,11 @@ 'GNU General Public License v3 or later (GPLv3+)', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12' ], ) diff --git a/tests/test_ipa_controller.py b/tests/test_ipa_controller.py index 77155d5..858e6a3 100644 --- a/tests/test_ipa_controller.py +++ b/tests/test_ipa_controller.py @@ -80,13 +80,13 @@ def test_controller_ec2_image(mock_test_image): @patch.object(IpaCloud, 'test_image') -@patch.object(GCECloud, '_get_driver') +@patch('img_proof.ipa_gce.compute_v1') @patch.object(GCECloud, '_get_credentials') @patch.object(GCECloud, '_validate_region') def test_controller_gce_image( mock_validate_region, mock_get_creds, - mock_get_driver, + mock_compute_v1, mock_test_image ): mock_test_image.return_value = (0, {'results': 'data'}) diff --git a/tests/test_ipa_gce.py b/tests/test_ipa_gce.py index 14d4692..6ce2010 100644 --- a/tests/test_ipa_gce.py +++ b/tests/test_ipa_gce.py @@ -21,30 +21,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import json import pytest from img_proof.ipa_gce import GCECloud -from img_proof.ipa_exceptions import GCECloudException, IpaRetryableError +from img_proof.ipa_exceptions import GCECloudException from unittest.mock import MagicMock, patch -from googleapiclient.errors import HttpError - - -def get_http_error(msg, status='404'): - resp = MagicMock() - resp.status = status - - content = { - 'error': { - 'code': int(status), - 'message': msg - } - } - - return HttpError(resp, json.dumps(content).encode()) - class TestGCECloud(object): """Test GCE cloud class.""" @@ -52,11 +35,11 @@ class TestGCECloud(object): @patch('img_proof.ipa_gce.AuthorizedSession') @patch('img_proof.ipa_gce.service_account') @patch.object(GCECloud, '_validate_region') - @patch('img_proof.ipa_gce.discovery') + @patch('img_proof.ipa_gce.compute_v1') def setup_method( self, method, - mock_discovery, + mock_compute_v1, mock_validate_region, mock_service_account, mock_auth_session @@ -74,8 +57,18 @@ def setup_method( } } - driver = MagicMock() - mock_discovery.build.return_value = driver + self.instances_client = MagicMock() + self.zone_ops_client = MagicMock() + self.zones_client = MagicMock() + self.networks_client = MagicMock() + self.subnet_client = MagicMock() + self.machine_type_client = MagicMock() + self.images_client = MagicMock() + self.disks_client = MagicMock() + mock_compute_v1.InstancesClient.return_value = self.instances_client + mock_compute_v1.ZoneOperationsClient.return_value = \ + self.zone_ops_client + mock_compute_v1.ZonesClient.return_value = self.zones_client service_account = MagicMock() mock_service_account.Credentials.\ @@ -142,18 +135,14 @@ def test_gce_get_creds_invalid( def test_gce_get_instance(self): """Test gce get instance method.""" instance = MagicMock() - instances_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = instance - instances_obj.get.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + self.instances_client.get.return_value = instance val = self.cloud._get_instance() assert val == instance self.cloud.running_instance_id = 'test-instance' - instances_obj.get.side_effect = get_http_error( + self.instances_client.get.side_effect = Exception( 'test-instance cannot be found.' ) @@ -163,20 +152,18 @@ def test_gce_get_instance(self): exc = "Unable to retrieve instance: test-instance cannot be found." assert str(error.value) == exc - def test_gce_get_network(self): + @patch('img_proof.ipa_gce.compute_v1') + def test_gce_get_network(self, mock_compute_v1): """Test GCE get network method.""" + mock_compute_v1.NetworksClient.return_value = self.networks_client network = MagicMock() - networks_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = network - networks_obj.get.return_value = operation - self.cloud.compute_driver.networks.return_value = networks_obj + self.networks_client.get.return_value = network result = self.cloud._get_network('test-network') assert result == network - networks_obj.get.side_effect = get_http_error( + self.networks_client.get.side_effect = Exception( 'Resource test-network not found.' ) @@ -186,21 +173,19 @@ def test_gce_get_network(self): assert msg == str(error.value) - def test_gce_get_subnet(self): + @patch('img_proof.ipa_gce.compute_v1') + def test_gce_get_subnet(self, mock_compute_v1): """Test GCE get subnetwork method.""" + mock_compute_v1.SubnetworksClient.return_value = self.subnet_client subnetwork = MagicMock() - subnet_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = subnetwork - subnet_obj.get.return_value = operation - self.cloud.compute_driver.subnetworks.return_value = subnet_obj + self.subnet_client.get.return_value = subnetwork self.cloud.region = 'us-west-1a' result = self.cloud._get_subnet('test-subnet') assert result == subnetwork - subnet_obj.get.side_effect = get_http_error( + self.subnet_client.get.side_effect = Exception( 'Resource test-subnet not found.' ) @@ -210,19 +195,18 @@ def test_gce_get_subnet(self): assert msg == str(error.value) - def test_gce_get_instance_type(self): + @patch('img_proof.ipa_gce.compute_v1') + def test_gce_get_instance_type(self, mock_compute_v1): """Test GCE get instance type method.""" + mock_compute_v1.MachineTypesClient.return_value = \ + self.machine_type_client machine_type = MagicMock() - machine_type_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = machine_type - machine_type_obj.get.return_value = operation - self.cloud.compute_driver.machineTypes.return_value = machine_type_obj + self.machine_type_client.get.return_value = machine_type result = self.cloud._get_instance_type('n1-standard-1') assert result == machine_type - machine_type_obj.get.side_effect = get_http_error( + self.machine_type_client.get.side_effect = Exception( 'Resource n1-standard-1 not found.' ) @@ -233,19 +217,17 @@ def test_gce_get_instance_type(self): assert msg == str(error.value) - def test_gce_get_image(self): + @patch('img_proof.ipa_gce.compute_v1') + def test_gce_get_image(self, mock_compute_v1): """Test GCE get image method.""" + mock_compute_v1.ImagesClient.return_value = self.images_client image = MagicMock() - image_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = image - image_obj.get.return_value = operation - self.cloud.compute_driver.images.return_value = image_obj + self.images_client.get.return_value = image result = self.cloud._get_image('fake-image-20200202') assert result == image - image_obj.get.side_effect = get_http_error( + self.images_client.get.side_effect = Exception( 'Resource fake-image-20200202 not found.' ) @@ -256,19 +238,17 @@ def test_gce_get_image(self): assert msg == str(error.value) - def test_gce_get_disk(self): + @patch('img_proof.ipa_gce.compute_v1') + def test_gce_get_disk(self, mock_compute_v1): """Test GCE get image method.""" + mock_compute_v1.DisksClient.return_value = self.disks_client disk = MagicMock() - disk_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = disk - disk_obj.get.return_value = operation - self.cloud.compute_driver.disks.return_value = disk_obj + self.disks_client.get.return_value = disk result = self.cloud._get_disk('disk12') assert result == disk - disk_obj.get.side_effect = get_http_error( + self.disks_client.get.side_effect = Exception( 'Resource disk12 not found.' ) @@ -281,28 +261,26 @@ def test_gce_get_disk(self): @patch.object(GCECloud, '_get_subnet') def test_get_network_config(self, mock_get_subnet): - subnet = 'projects/test/regions/us-west1/subnetworks/sub-123' - net = 'projects/test/global/networks/network' + subnet = MagicMock() + subnet.self_link = 'projects/test/regions/us-west1/subnetworks/sub-123' + subnet.network.self_link = 'projects/test/global/networks/network' - mock_get_subnet.return_value = { - 'selfLink': subnet, - 'network': net - } + mock_get_subnet.return_value = subnet subnet_config = self.cloud._get_network_config( 'sub-123', use_gvnic=True ) - assert subnet_config['network'] == net - assert subnet_config['subnetwork'] == subnet + assert subnet_config['network'] == subnet.network.self_link + assert subnet_config['subnetwork'] == subnet.self_link def test_get_shielded_instance_config(self): si_config = self.cloud.get_shielded_instance_config() - assert si_config['enableSecureBoot'] is False - assert si_config['enableVtpm'] - assert si_config['enableIntegrityMonitoring'] + assert si_config['enable_secure_boot'] is False + assert si_config['enable_vtpm'] + assert si_config['enable_integrity_monitoring'] def test_get_instance_config(self): config = self.cloud.get_instance_config( @@ -314,21 +292,21 @@ def test_get_instance_config(self): 'secretkey', 50, 'x86_64', - shielded_instance_config={'shielded': 'config'}, + shielded_instance_config={'enable_secure_boot': True}, sev='SEV_SNP', use_gvnic=True ) assert 'metadata' in config - assert 'serviceAccounts' in config - assert 'machineType' in config + assert 'service_accounts' in config + assert 'machine_type' in config assert 'disks' in config - assert 'networkInterfaces' in config + assert 'network_interfaces' in config assert 'name' in config - assert 'shieldedInstanceConfig' in config + assert 'shielded_instance_config' in config @patch.object(GCECloud, '_wait_on_instance') - @patch.object(GCECloud, '_wait_on_operation') + @patch.object(GCECloud, 'wait_for_extended_operation') @patch.object(GCECloud, '_get_network') @patch.object(GCECloud, '_get_image') @patch.object(GCECloud, '_get_instance_type') @@ -344,22 +322,23 @@ def test_gce_launch_instance( ): """Test GCE launch instance method.""" mock_generate_instance_name.return_value = 'test-instance' - mock_get_network.return_value = { - 'selfLink': 'projects/test/global/networks/net1' - } - mock_get_image.return_value = { - 'selfLink': 'projects/test/global/images/img-123' - } - mock_get_instance_type.return_value = { - 'selfLink': 'zones/us-west1-a/machineTypes/n1-standard-1' - } - mock_wait_on_operation.return_value = {} + network = MagicMock() + network.self_link = 'projects/test/global/networks/net1' + mock_get_network.return_value = network + + image = MagicMock() + image.self_link = 'projects/test/global/images/img-123' + mock_get_image.return_value = image + + inst_type = MagicMock() + inst_type.self_link = 'zones/us-west1-a/machineTypes/n1-standard-1' + mock_get_instance_type.return_value = inst_type + + mock_wait_on_operation.return_value = None - instances_obj = MagicMock() operation = MagicMock() - operation.execute.return_value = {'name': 'operation123'} - instances_obj.insert.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + operation.name = 'operation123' + self.instances_client.insert.return_value = operation self.cloud.region = 'us-west1-a' @@ -370,49 +349,29 @@ def test_gce_launch_instance( # Exception on operation - mock_wait_on_operation.return_value = { - 'error': { - 'errors': [{ - 'code': 'QUOTA_EXCEEDED', - 'message': 'Too many cpus.' - }] - } - } - - with pytest.raises(IpaRetryableError) as error: - self.cloud._launch_instance() - - assert 'Failed to launch instance: Too many cpus.' == str(error.value) - - # Exception on API call - - mock_wait_on_operation.return_value = {} - instances_obj.insert.side_effect = get_http_error( - 'Invalid instance type.', - '412' + self.instances_client.insert.side_effect = Exception( + 'Create failed!' ) - with pytest.raises(IpaRetryableError) as error: + with pytest.raises(GCECloudException) as error: self.cloud._launch_instance() - msg = 'Failed to launch instance: Invalid instance type.' - assert msg == str(error.value) + assert 'Failed to launch instance: Create failed!' == str(error.value) @patch.object(GCECloud, '_get_disk') @patch.object(GCECloud, '_get_instance') def test_gce_set_image_id(self, mock_get_instance, mock_get_disk): """Test gce cloud set image id method.""" - instance = { - 'disks': [{ - 'deviceName': 'disk123', - 'boot': True, - 'source': 'https://www.googleapis.com/compute/v1/projects/' - 'test/zones/us-west1-a/disks/disk123' - }] - } - disk = { - 'sourceImage': 'projects/suse/global/images/opensuse-leap-15.0' - } + instance = MagicMock() + disk = MagicMock() + disk.device_name = 'disk123' + disk.boot = True + disk.source = ( + 'https://www.googleapis.com/compute/v1/projects/' + 'test/zones/us-west1-a/disks/disk123' + ) + disk.source_image = 'projects/suse/global/images/opensuse-leap-15.0' + instance.disks = [disk] mock_get_instance.return_value = instance mock_get_disk.return_value = disk @@ -425,10 +384,8 @@ def test_gce_set_image_id(self, mock_get_instance, mock_get_disk): def test_gce_validate_region(self): """Test gce cloud set image id method.""" zones_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = None - zones_obj.get.return_value = operation - self.cloud.compute_driver.zones.return_value = zones_obj + zones_obj.return_value = None + self.zones_client.get.return_value = zones_obj with pytest.raises(GCECloudException) as error: self.cloud._validate_region() @@ -447,19 +404,25 @@ def test_gce_validate_region(self): @patch.object(GCECloud, '_get_instance') def test_gce_is_instance_running(self, mock_get_instance): """Test gce cloud is instance runnning method.""" - mock_get_instance.return_value = {'status': 'RUNNING'} + instance = MagicMock() + instance.status = 'RUNNING' + mock_get_instance.return_value = instance assert self.cloud._is_instance_running() assert mock_get_instance.call_count == 1 - mock_get_instance.return_value = {'status': 'TERMINATED'} + instance.status = 'TERMINATED' + mock_get_instance.return_value = instance assert not self.cloud._is_instance_running() @patch.object(GCECloud, '_get_instance') def test_gce_set_instance_ip(self, mock_get_instance): """Test gce cloud set instance ip method.""" - mock_get_instance.return_value = { - 'networkInterfaces': [{'some': 'data'}] - } + interface = MagicMock() + interface.network_i_p = None + interface.access_configs = [] + instance = MagicMock() + instance.network_interfaces = [interface] + mock_get_instance.return_value = instance self.cloud.running_instance_id = 'test' @@ -470,9 +433,8 @@ def test_gce_set_instance_ip(self, mock_get_instance): 'IP address for instance: test cannot be found.' assert mock_get_instance.call_count == 1 - mock_get_instance.return_value = { - 'networkInterfaces': [{'networkIP': '10.0.0.0'}] - } + interface.network_i_p = '10.0.0.0' + mock_get_instance.return_value = instance self.cloud._set_instance_ip() assert self.cloud.instance_ip == '10.0.0.0' @@ -481,53 +443,37 @@ def test_gce_set_instance_ip(self, mock_get_instance): def test_gce_start_instance(self, mock_wait_on_instance): """Test gce start instance method.""" mock_wait_on_instance.return_value = None - - instances_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = None - instances_obj.start.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + self.instances_client.start.return_value = None self.cloud._start_instance() - assert instances_obj.start.call_count == 1 + assert self.instances_client.start.call_count == 1 @patch.object(GCECloud, '_wait_on_instance') def test_gce_stop_instance(self, mock_wait_on_instance): """Test gce stop instance method.""" mock_wait_on_instance.return_value = None - - instances_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = None - instances_obj.stop.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + self.instances_client.stop.return_value = None self.cloud._stop_instance() - assert instances_obj.stop.call_count == 1 + assert self.instances_client.stop.call_count == 1 def test_gce_terminate_instance(self): """Test gce terminate instance method.""" - instances_obj = MagicMock() - operation = MagicMock() - operation.execute.return_value = None - instances_obj.delete.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + self.instances_client.delete.return_value = None self.cloud._terminate_instance() - assert instances_obj.delete.call_count == 1 + assert self.instances_client.delete.call_count == 1 def test_gce_get_console_log(self): """Test gce get console log method.""" - instances_obj = MagicMock() operation = MagicMock() - operation.execute.return_value = {'content': 'some output'} - instances_obj.getSerialPortOutput.return_value = operation - self.cloud.compute_driver.instances.return_value = instances_obj + operation.content = 'some output' + self.instances_client.get_serial_port_output.return_value = operation self.cloud.get_console_log() - assert instances_obj.getSerialPortOutput.call_count == 1 + assert self.instances_client.get_serial_port_output.call_count == 1 @patch('img_proof.ipa_gce.time') def test_wait_on_operation(self, mock_time): @@ -537,12 +483,24 @@ def test_wait_on_operation(self, mock_time): mock_time.sleep.return_value = None mock_time.time.return_value = 10 - zone_ops_obj = MagicMock() operation = MagicMock() - operation.execute.return_value = {'status': 'DONE'} - zone_ops_obj.get.return_value = operation - self.cloud.compute_driver.zoneOperations.return_value = zone_ops_obj + operation.status = 'DONE' + self.zone_ops_client.get.return_value = operation result = self.cloud._wait_on_operation('operation213') - assert result['status'] == 'DONE' - assert zone_ops_obj.get.call_count == 1 + assert result.status == 'DONE' + assert self.zone_ops_client.get.call_count == 1 + + def test_wait_for_extended_operation(self): + warning = MagicMock() + warning.code = '123' + warning.message = 'Something is wrong!' + + operation = MagicMock() + operation.error_code = '412' + operation.error_message = 'Operation failed!' + operation.warnings = [warning] + operation.result.return_value = False + + with pytest.raises(GCECloudException): + self.cloud.wait_for_extended_operation(operation)