diff --git a/.github/workflows/integration-informing.yaml b/.github/workflows/integration-informing.yaml index 344ffb6e4..f0eec21b8 100644 --- a/.github/workflows/integration-informing.yaml +++ b/.github/workflows/integration-informing.yaml @@ -89,6 +89,7 @@ jobs: TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap TEST_SUBSTRATE: lxd TEST_LXD_IMAGE: ${{ matrix.os }} + TEST_FLAVOR: ${{ matrix.patch }} TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports run: | # IPv6-only is only supported on moonray diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py index 98e170fae..bb62acaa4 100644 --- a/tests/integration/tests/conftest.py +++ b/tests/integration/tests/conftest.py @@ -1,9 +1,10 @@ # # Copyright 2024 Canonical, Ltd. # +import itertools import logging from pathlib import Path -from typing import Generator, List, Union +from typing import Generator, Iterator, List, Optional, Union import pytest from test_util import config, harness, util @@ -86,7 +87,8 @@ def pytest_configure(config): "no_setup: No setup steps (pushing snap, bootstrapping etc.) are performed on any node for this test.\n" "dualstack: Support dualstack on the instances.\n" "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n" - "node_count: Mark a test to specify how many instance nodes need to be created\n", + "node_count: Mark a test to specify how many instance nodes need to be created\n" + "snap_versions: Mark a test to specify snap_versions for each node\n", ) @@ -99,6 +101,15 @@ def node_count(request) -> int: return int(node_count_arg) +def snap_versions(request) -> Iterator[Optional[str]]: + """An endless iterable of snap versions for each node in the test.""" + marking = () + if snap_version_marker := request.node.get_closest_marker("snap_versions"): + marking, *_ = snap_version_marker.args + # endlessly repeat of the configured snap version after exhausting the marking + return itertools.chain(marking, itertools.repeat(None)) + + @pytest.fixture(scope="function") def disable_k8s_bootstrapping(request) -> bool: return bool(request.node.get_closest_marker("disable_k8s_bootstrapping")) @@ -132,28 +143,24 @@ def instances( no_setup: bool, bootstrap_config: Union[str, None], dualstack: bool, + request, ) -> Generator[List[harness.Instance], None, None]: """Construct instances for a cluster. Bootstrap and setup networking on the first instance, if `disable_k8s_bootstrapping` marker is not set. """ - if not config.SNAP: - pytest.fail("Set TEST_SNAP to the path where the snap is") - if node_count <= 0: pytest.xfail("Test requested 0 or fewer instances, skip this test.") - snap_path = (tmp_path / "k8s.snap").as_posix() - LOG.info(f"Creating {node_count} instances") instances: List[harness.Instance] = [] - for _ in range(node_count): + for _, snap in zip(range(node_count), snap_versions(request)): # Create instances and setup the k8s snap in each. instance = h.new_instance(dualstack=dualstack) instances.append(instance) if not no_setup: - util.setup_k8s_snap(instance, snap_path) + util.setup_k8s_snap(instance, tmp_path, snap) if not disable_k8s_bootstrapping and not no_setup: first_node, *_ = instances @@ -186,7 +193,7 @@ def instances( @pytest.fixture(scope="session") def session_instance( - h: harness.Harness, tmp_path_factory: pytest.TempPathFactory + h: harness.Harness, tmp_path_factory: pytest.TempPathFactory, request ) -> Generator[harness.Instance, None, None]: """Constructs and bootstraps an instance that persists over a test session. @@ -194,9 +201,10 @@ def session_instance( """ LOG.info("Setup node and enable all features") - snap_path = str(tmp_path_factory.mktemp("data") / "k8s.snap") + tmp_path = tmp_path_factory.mktemp("data") instance = h.new_instance() - util.setup_k8s_snap(instance, snap_path) + snap = next(snap_versions(request)) + util.setup_k8s_snap(instance, tmp_path, snap) bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml" instance.send_file( diff --git a/tests/integration/tests/test_cleanup.py b/tests/integration/tests/test_cleanup.py index e3fa4e37e..fb19a1ebf 100644 --- a/tests/integration/tests/test_cleanup.py +++ b/tests/integration/tests/test_cleanup.py @@ -5,7 +5,7 @@ from typing import List import pytest -from test_util import harness, util +from test_util import config, harness, util LOG = logging.getLogger(__name__) @@ -17,7 +17,7 @@ def test_node_cleanup(instances: List[harness.Instance]): util.wait_for_network(instance) LOG.info("Uninstall k8s...") - instance.exec(["snap", "remove", "k8s", "--purge"]) + instance.exec(["snap", "remove", config.SNAP_NAME, "--purge"]) LOG.info("Waiting for shims to go away...") util.stubbornly(retries=5, delay_s=5).on(instance).until( diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py index 4650b15b9..a77e3f9c5 100644 --- a/tests/integration/tests/test_clustering.py +++ b/tests/integration/tests/test_clustering.py @@ -33,6 +33,31 @@ def test_control_plane_nodes(instances: List[harness.Instance]): ), f"only {cluster_node.id} should be left in cluster" +@pytest.mark.node_count(2) +@pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP]) +def test_mixed_version_join(instances: List[harness.Instance]): + """Test n versioned node joining a n-1 versioned cluster.""" + cluster_node = instances[0] # bootstrapped on the previous channel + joining_node = instances[1] # installed with the snap under test + + join_token = util.get_join_token(cluster_node, joining_node) + util.join_cluster(joining_node, join_token) + + util.wait_until_k8s_ready(cluster_node, instances) + nodes = util.ready_nodes(cluster_node) + assert len(nodes) == 2, "node should have joined cluster" + + assert "control-plane" in util.get_local_node_status(cluster_node) + assert "control-plane" in util.get_local_node_status(joining_node) + + cluster_node.exec(["k8s", "remove-node", joining_node.id]) + nodes = util.ready_nodes(cluster_node) + assert len(nodes) == 1, "node should have been removed from cluster" + assert ( + nodes[0]["metadata"]["name"] == cluster_node.id + ), f"only {cluster_node.id} should be left in cluster" + + @pytest.mark.node_count(3) def test_worker_nodes(instances: List[harness.Instance]): cluster_node = instances[0] diff --git a/tests/integration/tests/test_util/config.py b/tests/integration/tests/test_util/config.py index fddef4e2a..45c9ea5e6 100644 --- a/tests/integration/tests/test_util/config.py +++ b/tests/integration/tests/test_util/config.py @@ -17,9 +17,15 @@ # ETCD_VERSION is the version of etcd to use. ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.4.34" +# FLAVOR is the flavor of the snap to use. +FLAVOR = os.getenv("TEST_FLAVOR") or "" + # SNAP is the absolute path to the snap against which we run the integration tests. SNAP = os.getenv("TEST_SNAP") +# SNAP_NAME is the name of the snap under test. +SNAP_NAME = os.getenv("TEST_SNAP_NAME") or "k8s" + # SUBSTRATE is the substrate to use for running the integration tests. # One of 'local' (default), 'lxd', 'juju', or 'multipass'. SUBSTRATE = os.getenv("TEST_SUBSTRATE") or "local" diff --git a/tests/integration/tests/test_util/harness/base.py b/tests/integration/tests/test_util/harness/base.py index 8111a339d..de12508c6 100644 --- a/tests/integration/tests/test_util/harness/base.py +++ b/tests/integration/tests/test_util/harness/base.py @@ -32,6 +32,7 @@ def id(self) -> str: @cached_property def arch(self) -> str: + """Return the architecture of the instance""" return self.exec( ["dpkg", "--print-architecture"], text=True, capture_output=True ).stdout.strip() diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 0124b5f84..391d73f6a 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -6,11 +6,13 @@ import re import shlex import subprocess +import urllib.request from datetime import datetime from functools import partial from pathlib import Path from typing import Any, Callable, List, Mapping, Optional, Union +import pytest from tenacity import ( RetryCallState, retry, @@ -22,6 +24,8 @@ from test_util import config, harness LOG = logging.getLogger(__name__) +RISKS = ["stable", "candidate", "beta", "edge"] +TRACK_RE = re.compile(r"^(\d+)\.(\d+)(\S*)$") def run(command: list, **kwargs) -> subprocess.CompletedProcess: @@ -128,12 +132,50 @@ def until( return Retriable() -# Installs and setups the k8s snap on the given instance and connects the interfaces. -def setup_k8s_snap(instance: harness.Instance, snap_path: Path): - LOG.info("Install k8s snap") - instance.send_file(config.SNAP, snap_path) - instance.exec(["snap", "install", snap_path, "--classic", "--dangerous"]) +def _as_int(value: Optional[str]) -> Optional[int]: + """Convert a string to an integer.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + +def setup_k8s_snap( + instance: harness.Instance, tmp_path: Path, snap: Optional[str] = None +): + """Installs and sets up the snap on the given instance and connects the interfaces. + Args: + instance: instance on which to install the snap + tmp_path: path to store the snap on the instance + snap: choice of track, channel, revision, or file path + a snap track to install + a snap channel to install + a snap revision to install + a path to the snap to install + """ + cmd = ["snap", "install", "--classic"] + which_snap = snap or config.SNAP + + if not which_snap: + pytest.fail("Set TEST_SNAP to the channel, revision, or path to the snap") + + if isinstance(which_snap, str) and which_snap.startswith("/"): + LOG.info("Install k8s snap by path") + snap_path = (tmp_path / "k8s.snap").as_posix() + instance.send_file(which_snap, snap_path) + cmd += ["--dangerous", snap_path] + elif snap_revision := _as_int(which_snap): + LOG.info("Install k8s snap by revision") + cmd += [config.SNAP_NAME, "--revision", snap_revision] + elif "/" in which_snap or which_snap in RISKS: + LOG.info("Install k8s snap by specific channel: %s", which_snap) + cmd += [config.SNAP_NAME, "--channel", which_snap] + elif channel := tracks_least_risk(which_snap, instance.arch): + LOG.info("Install k8s snap by least risky channel: %s", channel) + cmd += [config.SNAP_NAME, "--channel", channel] + + instance.exec(cmd) LOG.info("Ensure k8s interfaces and network requirements") instance.exec(["/snap/k8s/current/k8s/hack/init.sh"], stdout=subprocess.DEVNULL) @@ -294,3 +336,95 @@ def is_valid_rfc3339(date_str): return True except ValueError: return False + + +def tracks_least_risk(track: str, arch: str) -> str: + """Determine the snap channel with the least risk in the provided track. + + Args: + track: the track to determine the least risk channel for + arch: the architecture to narrow the revision + + Returns: + the channel associated with the least risk + """ + LOG.debug("Determining least risk channel for track: %s on %s", track, arch) + if track == "latest": + return f"latest/edge/{config.FLAVOR or 'classic'}" + + INFO_URL = f"https://api.snapcraft.io/v2/snaps/info/{config.SNAP_NAME}" + HEADERS = { + "Snap-Device-Series": "16", + "User-Agent": "Mozilla/5.0", + } + + req = urllib.request.Request(INFO_URL, headers=HEADERS) + with urllib.request.urlopen(req) as response: + snap_info = json.loads(response.read().decode()) + + risks = [ + channel["channel"]["risk"] + for channel in snap_info["channel-map"] + if channel["channel"]["track"] == track + and channel["channel"]["architecture"] == arch + ] + if not risks: + raise ValueError(f"No risks found for track: {track}") + risk_level = {"stable": 0, "candidate": 1, "beta": 2, "edge": 3} + channel = f"{track}/{min(risks, key=lambda r: risk_level[r])}" + LOG.info("Least risk channel from track %s is %s", track, channel) + return channel + + +def previous_track(snap_version: str) -> str: + """Determine the snap track preceding the provided version. + + Args: + snap_version: the snap version to determine the previous track for + + Returns: + the previous track + """ + LOG.debug("Determining previous track for %s", snap_version) + + def _maj_min(version: str): + if match := TRACK_RE.match(version): + maj, min, _ = match.groups() + return int(maj), int(min) + return None + + if not snap_version: + assumed = "latest" + LOG.info( + "Cannot determine previous track for undefined snap -- assume %s", + snap_version, + assumed, + ) + return assumed + + if snap_version.startswith("/") or _as_int(snap_version) is not None: + assumed = "latest" + LOG.info( + "Cannot determine previous track for %s -- assume %s", snap_version, assumed + ) + return assumed + + if maj_min := _maj_min(snap_version): + maj, min = maj_min + if min == 0: + with urllib.request.urlopen( + f"https://dl.k8s.io/release/stable-{maj - 1}.txt" + ) as r: + stable = r.read().decode().strip() + maj_min = _maj_min(stable) + else: + maj_min = (maj, min - 1) + elif snap_version.startswith("latest") or "/" not in snap_version: + with urllib.request.urlopen("https://dl.k8s.io/release/stable.txt") as r: + stable = r.read().decode().strip() + maj_min = _maj_min(stable) + + flavor_track = {"": "classic", "strict": ""}.get(config.FLAVOR, config.FLAVOR) + track = f"{maj_min[0]}.{maj_min[1]}" + (flavor_track and f"-{flavor_track}") + LOG.info("Previous track for %s is from track: %s", snap_version, track) + return track diff --git a/tests/integration/tests/test_version_upgrades.py b/tests/integration/tests/test_version_upgrades.py index d6e7a45b0..3d9207759 100644 --- a/tests/integration/tests/test_version_upgrades.py +++ b/tests/integration/tests/test_version_upgrades.py @@ -12,40 +12,45 @@ @pytest.mark.node_count(1) @pytest.mark.no_setup() -@pytest.mark.xfail(reason="cilium failures are blocking this from working") @pytest.mark.skipif( not config.VERSION_UPGRADE_CHANNELS, reason="No upgrade channels configured" ) -def test_version_upgrades(instances: List[harness.Instance]): +def test_version_upgrades(instances: List[harness.Instance], tmp_path): channels = config.VERSION_UPGRADE_CHANNELS cp = instances[0] + current_channel = channels[0] - if channels[0].lower() == "recent": + if current_channel.lower() == "recent": if len(channels) != 3: pytest.fail( "'recent' requires the number of releases as second argument and the flavour as third argument" ) _, num_channels, flavour = channels channels = snap.get_latest_channels(int(num_channels), flavour, cp.arch) + current_channel = channels[0] LOG.info( - f"Bootstrap node on {channels[0]} and upgrade through channels: {channels[1:]}" + f"Bootstrap node on {current_channel} and upgrade through channels: {channels[1:]}" ) # Setup the k8s snap from the bootstrap channel and setup basic configuration. - cp.exec(["snap", "install", "k8s", "--channel", channels[0], "--classic"]) + util.setup_k8s_snap(cp, tmp_path, current_channel) cp.exec(["k8s", "bootstrap"]) - util.stubbornly(retries=30, delay_s=20).until(util.ready_nodes(cp) == 1) + util.wait_until_k8s_ready(cp, instances) + LOG.info(f"Installed {cp.id} on channel {current_channel}") - current_channel = channels[0] for channel in channels[1:]: LOG.info(f"Upgrading {cp.id} from {current_channel} to channel {channel}") + # Log the current snap version on the node. - cp.exec(["snap", "info", "k8s"]) + out = cp.exec(["snap", "list", config.SNAP_NAME], capture_output=True) + LOG.info(f"Current snap version: {out.stdout.decode().strip()}") # note: the `--classic` flag will be ignored by snapd for strict snaps. - cp.exec(["snap", "refresh", "k8s", "--channel", channel, "--classic"]) - - util.stubbornly(retries=30, delay_s=20).until(util.ready_nodes(cp) == 1) - LOG.info(f"Upgraded {cp.id} to channel {channel}") + cp.exec( + ["snap", "refresh", config.SNAP_NAME, "--channel", channel, "--classic"] + ) + util.wait_until_k8s_ready(cp, instances) + current_channel = channel + LOG.info(f"Upgraded {cp.id} on channel {channel}")