From 3a5b5872c5418317b58dbf2299020c75828330a7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 7 Jan 2022 15:52:09 +0000 Subject: [PATCH] Add package-upgrade action The package-upgrade action performs package upgrades for the current OpenStack release. The code path used is similar to the openstack-upgrade action, with the difference being that package-upgrade will not execute if an openstack upgrade is available (based on the openstack-origin setting). This change includes a charm-helpers sync. Change-Id: I0c7184bba29731354e52dc28e3a4dd6f282fa843 --- actions.yaml | 3 + actions/package-upgrade | 1 + actions/package_upgrade.py | 62 +++++++++++++++++++ charmhelpers/contrib/hahelpers/cluster.py | 7 +++ .../contrib/openstack/deferred_events.py | 4 +- .../section-keystone-authtoken-mitaka | 2 + charmhelpers/contrib/openstack/utils.py | 2 +- charmhelpers/contrib/storage/linux/ceph.py | 23 +++++-- charmhelpers/fetch/ubuntu.py | 2 +- unit_tests/test_actions_package_upgrade.py | 62 +++++++++++++++++++ 10 files changed, 160 insertions(+), 8 deletions(-) create mode 120000 actions/package-upgrade create mode 100755 actions/package_upgrade.py create mode 100644 unit_tests/test_actions_package_upgrade.py diff --git a/actions.yaml b/actions.yaml index c952b74b..38f975dd 100644 --- a/actions.yaml +++ b/actions.yaml @@ -4,3 +4,6 @@ resume: description: Resume the swift-storage unit. This action will start Swift services. openstack-upgrade: description: Perform openstack upgrades. Config option action-managed-upgrade must be set to True. +package-upgrade: + description: | + Perform package upgrades for the current OpenStack release. diff --git a/actions/package-upgrade b/actions/package-upgrade new file mode 120000 index 00000000..6c6a7b84 --- /dev/null +++ b/actions/package-upgrade @@ -0,0 +1 @@ +package_upgrade.py \ No newline at end of file diff --git a/actions/package_upgrade.py b/actions/package_upgrade.py new file mode 100755 index 00000000..587d683e --- /dev/null +++ b/actions/package_upgrade.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 +# +# Copyright 2022 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +_path = os.path.dirname(os.path.realpath(__file__)) +_root = os.path.abspath(os.path.join(_path, '..')) + + +def _add_path(path): + if path not in sys.path: + sys.path.insert(1, path) + + +_add_path(_root) + + +from charmhelpers.contrib.openstack.utils import ( + do_action_package_upgrade, +) + +from hooks.swift_storage_hooks import ( + config_changed, + CONFIGS, +) + +from lib.swift_storage_utils import ( + do_openstack_upgrade, +) + + +def package_upgrade(): + """Perform package upgrade within the current OpenStack release. + + In order to prevent this action from upgrading to a new release of + OpenStack, package upgrades are not run if a new OpenStack release is + available. See source of do_action_package_upgrade() for this check. + + Upgrades packages and sets the corresponding action status as a result.""" + + if (do_action_package_upgrade('swift', + do_openstack_upgrade, + CONFIGS)): + config_changed() + + +if __name__ == '__main__': + package_upgrade() diff --git a/charmhelpers/contrib/hahelpers/cluster.py b/charmhelpers/contrib/hahelpers/cluster.py index ffda5fe1..7b309256 100644 --- a/charmhelpers/contrib/hahelpers/cluster.py +++ b/charmhelpers/contrib/hahelpers/cluster.py @@ -221,6 +221,13 @@ def https(): return True if config_get('ssl_cert') and config_get('ssl_key'): return True + # Local import to avoid ciruclar dependency. + import charmhelpers.contrib.openstack.cert_utils as cert_utils + if ( + cert_utils.get_certificate_request() and not + cert_utils.get_requests_for_local_unit("certificates") + ): + return False for r_id in relation_ids('certificates'): for unit in relation_list(r_id): ca = relation_get('ca', rid=r_id, unit=unit) diff --git a/charmhelpers/contrib/openstack/deferred_events.py b/charmhelpers/contrib/openstack/deferred_events.py index 94eacf6c..4c46e41a 100644 --- a/charmhelpers/contrib/openstack/deferred_events.py +++ b/charmhelpers/contrib/openstack/deferred_events.py @@ -127,7 +127,9 @@ def deferred_events(): """ events = [] for defer_file in deferred_events_files(): - events.append((defer_file, read_event_file(defer_file))) + event = read_event_file(defer_file) + if event.policy_requestor_name == hookenv.service_name(): + events.append((defer_file, event)) return events diff --git a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka index 14c25b4d..139a0512 100644 --- a/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka +++ b/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -22,4 +22,6 @@ signing_dir = {{ signing_dir }} {% if use_memcache == true %} memcached_servers = {{ memcache_url }} {% endif -%} +service_token_roles = {{ admin_role }} +service_token_roles_required = True {% endif -%} diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index 3d52eb16..83b6884b 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -957,7 +957,7 @@ def os_requires_version(ostack_release, pkg): def wrap(f): @wraps(f) def wrapped_f(*args): - if os_release(pkg) < ostack_release: + if CompareOpenStackReleases(os_release(pkg)) < ostack_release: raise Exception("This hook is not supported on releases" " before %s" % ostack_release) f(*args) diff --git a/charmhelpers/contrib/storage/linux/ceph.py b/charmhelpers/contrib/storage/linux/ceph.py index 1b20b8fe..2e1fc1b5 100644 --- a/charmhelpers/contrib/storage/linux/ceph.py +++ b/charmhelpers/contrib/storage/linux/ceph.py @@ -28,7 +28,6 @@ import shutil import json import time -import uuid from subprocess import ( check_call, @@ -1677,6 +1676,10 @@ class CephBrokerRq(object): The API is versioned and defaults to version 1. """ + # The below hash is the result of running + # `hashlib.sha1('[]'.encode()).hexdigest()` + EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c' + def __init__(self, api_version=1, request_id=None, raw_request_data=None): """Initialize CephBrokerRq object. @@ -1685,8 +1688,12 @@ def __init__(self, api_version=1, request_id=None, raw_request_data=None): :param api_version: API version for request (default: 1). :type api_version: Optional[int] - :param request_id: Unique identifier for request. - (default: string representation of generated UUID) + :param request_id: Unique identifier for request. The identifier will + be updated as ops are added or removed from the + broker request. This ensures that Ceph will + correctly process requests where operations are + added after the initial request is processed. + (default: sha1 of operations) :type request_id: Optional[str] :param raw_request_data: JSON-encoded string to build request from. :type raw_request_data: Optional[str] @@ -1695,16 +1702,20 @@ def __init__(self, api_version=1, request_id=None, raw_request_data=None): if raw_request_data: request_data = json.loads(raw_request_data) self.api_version = request_data['api-version'] - self.request_id = request_data['request-id'] self.set_ops(request_data['ops']) + self.request_id = request_data['request-id'] else: self.api_version = api_version if request_id: self.request_id = request_id else: - self.request_id = str(uuid.uuid1()) + self.request_id = CephBrokerRq.EMPTY_LIST_SHA self.ops = [] + def _hash_ops(self): + """Return the sha1 of the requested Broker ops.""" + return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest() + def add_op(self, op): """Add an op if it is not already in the list. @@ -1713,6 +1724,7 @@ def add_op(self, op): """ if op not in self.ops: self.ops.append(op) + self.request_id = self._hash_ops() def add_op_request_access_to_group(self, name, namespace=None, permission=None, key_name=None, @@ -1991,6 +2003,7 @@ def set_ops(self, ops): to allow comparisons to ensure validity. """ self.ops = ops + self.request_id = self._hash_ops() @property def request(self): diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index effc884a..1bad0db8 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -591,7 +591,7 @@ def _get_key_by_keyid(keyid): curl_cmd = ['curl', keyserver_url.format(keyid)] # use proxy server settings in order to retrieve the key return subprocess.check_output(curl_cmd, - env=env_proxy_settings(['https'])) + env=env_proxy_settings(['https', 'no_proxy'])) def _dearmor_gpg_key(key_asc): diff --git a/unit_tests/test_actions_package_upgrade.py b/unit_tests/test_actions_package_upgrade.py new file mode 100644 index 00000000..a3374960 --- /dev/null +++ b/unit_tests/test_actions_package_upgrade.py @@ -0,0 +1,62 @@ +# Copyright 2022 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from unittest.mock import patch, MagicMock + +os.environ['JUJU_UNIT_NAME'] = 'swift-storage' + +# python-apt is not installed as part of test-requirements but is imported by +# some charmhelpers modules so create a fake import. +sys.modules['apt'] = MagicMock() +sys.modules['apt_pkg'] = MagicMock() + +with patch('charmhelpers.contrib.hardening.harden.harden') as mock_dec: + mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f: + lambda *args, **kwargs: f(*args, **kwargs)) + with patch('lib.misc_utils.is_paused') as is_paused: + with patch('lib.swift_storage_utils.register_configs'): + import actions.package_upgrade as package_upgrade + +from unit_tests.test_utils import CharmTestCase + +TO_PATCH = [ + 'config_changed', + 'do_openstack_upgrade', +] + + +class TestSwiftStorageUpgradeActions(CharmTestCase): + + def setUp(self): + super(TestSwiftStorageUpgradeActions, self).setUp(package_upgrade, + TO_PATCH) + + @patch('charmhelpers.contrib.openstack.utils.action_set') + @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available') + def test_package_upgrade_success(self, upgrade_avail, + action_set): + upgrade_avail.return_value = False + package_upgrade.package_upgrade() + self.assertTrue(self.do_openstack_upgrade.called) + + @patch('charmhelpers.contrib.openstack.utils.action_set') + @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available') + def test_package_upgrade_failure(self, upgrade_avail, + action_set): + upgrade_avail.return_value = True + package_upgrade.package_upgrade() + self.assertFalse(self.do_openstack_upgrade.called)