Create a test scenario for mixed version clustering (#720)

.github/workflows/integration-informing.yaml

@@ -89,6 +89,7 @@ jobs:
           TEST_SNAP: ${{ github.workspace }}/build/k8s-${{ matrix.patch }}.snap
           TEST_SUBSTRATE: lxd
           TEST_LXD_IMAGE: ${{ matrix.os }}
+          TEST_FLAVOR: ${{ matrix.patch }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
         run: |
           # IPv6-only is only supported on moonray

tests/integration/tests/conftest.py

@@ -1,9 +1,10 @@
 #
 # Copyright 2024 Canonical, Ltd.
 #
+import itertools
 import logging
 from pathlib import Path
-from typing import Generator, List, Union
+from typing import Generator, Iterator, List, Optional, Union
 
 import pytest
 from test_util import config, harness, util
@@ -86,7 +87,8 @@ def pytest_configure(config):
         "no_setup: No setup steps (pushing snap, bootstrapping etc.) are performed on any node for this test.\n"
         "dualstack: Support dualstack on the instances.\n"
         "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)\n"
-        "node_count: Mark a test to specify how many instance nodes need to be created\n",
+        "node_count: Mark a test to specify how many instance nodes need to be created\n"
+        "snap_versions: Mark a test to specify snap_versions for each node\n",
     )
 
 
@@ -99,6 +101,15 @@ def node_count(request) -> int:
     return int(node_count_arg)
 
 
+def snap_versions(request) -> Iterator[Optional[str]]:
+    """An endless iterable of snap versions for each node in the test."""
+    marking = ()
+    if snap_version_marker := request.node.get_closest_marker("snap_versions"):
+        marking, *_ = snap_version_marker.args
+    # endlessly repeat of the configured snap version after exhausting the marking
+    return itertools.chain(marking, itertools.repeat(None))
+
+
 @pytest.fixture(scope="function")
 def disable_k8s_bootstrapping(request) -> bool:
     return bool(request.node.get_closest_marker("disable_k8s_bootstrapping"))
@@ -132,28 +143,24 @@ def instances(
     no_setup: bool,
     bootstrap_config: Union[str, None],
     dualstack: bool,
+    request,
 ) -> Generator[List[harness.Instance], None, None]:
     """Construct instances for a cluster.
 
     Bootstrap and setup networking on the first instance, if `disable_k8s_bootstrapping` marker is not set.
     """
-    if not config.SNAP:
-        pytest.fail("Set TEST_SNAP to the path where the snap is")
-
     if node_count <= 0:
         pytest.xfail("Test requested 0 or fewer instances, skip this test.")
 
-    snap_path = (tmp_path / "k8s.snap").as_posix()
-
     LOG.info(f"Creating {node_count} instances")
     instances: List[harness.Instance] = []
 
-    for _ in range(node_count):
+    for _, snap in zip(range(node_count), snap_versions(request)):
         # Create <node_count> instances and setup the k8s snap in each.
         instance = h.new_instance(dualstack=dualstack)
         instances.append(instance)
         if not no_setup:
-            util.setup_k8s_snap(instance, snap_path)
+            util.setup_k8s_snap(instance, tmp_path, snap)
 
     if not disable_k8s_bootstrapping and not no_setup:
         first_node, *_ = instances
@@ -186,17 +193,18 @@ def instances(
 
 @pytest.fixture(scope="session")
 def session_instance(
-    h: harness.Harness, tmp_path_factory: pytest.TempPathFactory
+    h: harness.Harness, tmp_path_factory: pytest.TempPathFactory, request
 ) -> Generator[harness.Instance, None, None]:
     """Constructs and bootstraps an instance that persists over a test session.
 
     Bootstraps the instance with all k8sd features enabled to reduce testing time.
     """
     LOG.info("Setup node and enable all features")
 
-    snap_path = str(tmp_path_factory.mktemp("data") / "k8s.snap")
+    tmp_path = tmp_path_factory.mktemp("data")
     instance = h.new_instance()
-    util.setup_k8s_snap(instance, snap_path)
+    snap = next(snap_versions(request))
+    util.setup_k8s_snap(instance, tmp_path, snap)
 
     bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml"
     instance.send_file(

tests/integration/tests/test_cleanup.py

@@ -5,7 +5,7 @@
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import config, harness, util
 
 LOG = logging.getLogger(__name__)
 
@@ -17,7 +17,7 @@ def test_node_cleanup(instances: List[harness.Instance]):
     util.wait_for_network(instance)
 
     LOG.info("Uninstall k8s...")
-    instance.exec(["snap", "remove", "k8s", "--purge"])
+    instance.exec(["snap", "remove", config.SNAP_NAME, "--purge"])
 
     LOG.info("Waiting for shims to go away...")
     util.stubbornly(retries=5, delay_s=5).on(instance).until(

tests/integration/tests/test_clustering.py

@@ -33,6 +33,31 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
     ), f"only {cluster_node.id} should be left in cluster"
 
 
+@pytest.mark.node_count(2)
+@pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
+def test_mixed_version_join(instances: List[harness.Instance]):
+    """Test n versioned node joining a n-1 versioned cluster."""
+    cluster_node = instances[0]  # bootstrapped on the previous channel
+    joining_node = instances[1]  # installed with the snap under test
+
+    join_token = util.get_join_token(cluster_node, joining_node)
+    util.join_cluster(joining_node, join_token)
+
+    util.wait_until_k8s_ready(cluster_node, instances)
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 2, "node should have joined cluster"
+
+    assert "control-plane" in util.get_local_node_status(cluster_node)
+    assert "control-plane" in util.get_local_node_status(joining_node)
+
+    cluster_node.exec(["k8s", "remove-node", joining_node.id])
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 1, "node should have been removed from cluster"
+    assert (
+        nodes[0]["metadata"]["name"] == cluster_node.id
+    ), f"only {cluster_node.id} should be left in cluster"
+
+
 @pytest.mark.node_count(3)
 def test_worker_nodes(instances: List[harness.Instance]):
     cluster_node = instances[0]

tests/integration/tests/test_util/config.py

@@ -17,9 +17,15 @@
 # ETCD_VERSION is the version of etcd to use.
 ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.4.34"
 
+# FLAVOR is the flavor of the snap to use.
+FLAVOR = os.getenv("TEST_FLAVOR") or ""
+
 # SNAP is the absolute path to the snap against which we run the integration tests.
 SNAP = os.getenv("TEST_SNAP")
 
+# SNAP_NAME is the name of the snap under test.
+SNAP_NAME = os.getenv("TEST_SNAP_NAME") or "k8s"
+
 # SUBSTRATE is the substrate to use for running the integration tests.
 # One of 'local' (default), 'lxd', 'juju', or 'multipass'.
 SUBSTRATE = os.getenv("TEST_SUBSTRATE") or "local"

tests/integration/tests/test_util/harness/base.py

@@ -32,6 +32,7 @@ def id(self) -> str:
 
     @cached_property
     def arch(self) -> str:
+        """Return the architecture of the instance"""
         return self.exec(
             ["dpkg", "--print-architecture"], text=True, capture_output=True
         ).stdout.strip()

tests/integration/tests/test_util/util.py

@@ -6,11 +6,13 @@
 import re
 import shlex
 import subprocess
+import urllib.request
 from datetime import datetime
 from functools import partial
 from pathlib import Path
 from typing import Any, Callable, List, Mapping, Optional, Union
 
+import pytest
 from tenacity import (
     RetryCallState,
     retry,
@@ -22,6 +24,8 @@
 from test_util import config, harness
 
 LOG = logging.getLogger(__name__)
+RISKS = ["stable", "candidate", "beta", "edge"]
+TRACK_RE = re.compile(r"^(\d+)\.(\d+)(\S*)$")
 
 
 def run(command: list, **kwargs) -> subprocess.CompletedProcess:
@@ -128,12 +132,50 @@ def until(
     return Retriable()
 
 
-# Installs and setups the k8s snap on the given instance and connects the interfaces.
-def setup_k8s_snap(instance: harness.Instance, snap_path: Path):
-    LOG.info("Install k8s snap")
-    instance.send_file(config.SNAP, snap_path)
-    instance.exec(["snap", "install", snap_path, "--classic", "--dangerous"])
+def _as_int(value: Optional[str]) -> Optional[int]:
+    """Convert a string to an integer."""
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def setup_k8s_snap(
+    instance: harness.Instance, tmp_path: Path, snap: Optional[str] = None
+):
+    """Installs and sets up the snap on the given instance and connects the interfaces.
 
+    Args:
+        instance:   instance on which to install the snap
+        tmp_path:   path to store the snap on the instance
+        snap: choice of track, channel, revision, or file path
+            a snap track to install
+            a snap channel to install
+            a snap revision to install
+            a path to the snap to install
+    """
+    cmd = ["snap", "install", "--classic"]
+    which_snap = snap or config.SNAP
+
+    if not which_snap:
+        pytest.fail("Set TEST_SNAP to the channel, revision, or path to the snap")
+
+    if isinstance(which_snap, str) and which_snap.startswith("/"):
+        LOG.info("Install k8s snap by path")
+        snap_path = (tmp_path / "k8s.snap").as_posix()
+        instance.send_file(which_snap, snap_path)
+        cmd += ["--dangerous", snap_path]
+    elif snap_revision := _as_int(which_snap):
+        LOG.info("Install k8s snap by revision")
+        cmd += [config.SNAP_NAME, "--revision", snap_revision]
+    elif "/" in which_snap or which_snap in RISKS:
+        LOG.info("Install k8s snap by specific channel: %s", which_snap)
+        cmd += [config.SNAP_NAME, "--channel", which_snap]
+    elif channel := tracks_least_risk(which_snap, instance.arch):
+        LOG.info("Install k8s snap by least risky channel: %s", channel)
+        cmd += [config.SNAP_NAME, "--channel", channel]
+
+    instance.exec(cmd)
     LOG.info("Ensure k8s interfaces and network requirements")
     instance.exec(["/snap/k8s/current/k8s/hack/init.sh"], stdout=subprocess.DEVNULL)
 
@@ -294,3 +336,95 @@ def is_valid_rfc3339(date_str):
         return True
     except ValueError:
         return False
+
+
+def tracks_least_risk(track: str, arch: str) -> str:
+    """Determine the snap channel with the least risk in the provided track.
+
+    Args:
+        track: the track to determine the least risk channel for
+        arch: the architecture to narrow the revision
+
+    Returns:
+        the channel associated with the least risk
+    """
+    LOG.debug("Determining least risk channel for track: %s on %s", track, arch)
+    if track == "latest":
+        return f"latest/edge/{config.FLAVOR or 'classic'}"
+
+    INFO_URL = f"https://api.snapcraft.io/v2/snaps/info/{config.SNAP_NAME}"
+    HEADERS = {
+        "Snap-Device-Series": "16",
+        "User-Agent": "Mozilla/5.0",
+    }
+
+    req = urllib.request.Request(INFO_URL, headers=HEADERS)
+    with urllib.request.urlopen(req) as response:
+        snap_info = json.loads(response.read().decode())
+
+    risks = [
+        channel["channel"]["risk"]
+        for channel in snap_info["channel-map"]
+        if channel["channel"]["track"] == track
+        and channel["channel"]["architecture"] == arch
+    ]
+    if not risks:
+        raise ValueError(f"No risks found for track: {track}")
+    risk_level = {"stable": 0, "candidate": 1, "beta": 2, "edge": 3}
+    channel = f"{track}/{min(risks, key=lambda r: risk_level[r])}"
+    LOG.info("Least risk channel from track %s is %s", track, channel)
+    return channel
+
+
+def previous_track(snap_version: str) -> str:
+    """Determine the snap track preceding the provided version.
+
+    Args:
+        snap_version: the snap version to determine the previous track for
+
+    Returns:
+        the previous track
+    """
+    LOG.debug("Determining previous track for %s", snap_version)
+
+    def _maj_min(version: str):
+        if match := TRACK_RE.match(version):
+            maj, min, _ = match.groups()
+            return int(maj), int(min)
+        return None
+
+    if not snap_version:
+        assumed = "latest"
+        LOG.info(
+            "Cannot determine previous track for undefined snap -- assume %s",
+            snap_version,
+            assumed,
+        )
+        return assumed
+
+    if snap_version.startswith("/") or _as_int(snap_version) is not None:
+        assumed = "latest"
+        LOG.info(
+            "Cannot determine previous track for %s -- assume %s", snap_version, assumed
+        )
+        return assumed
+
+    if maj_min := _maj_min(snap_version):
+        maj, min = maj_min
+        if min == 0:
+            with urllib.request.urlopen(
+                f"https://dl.k8s.io/release/stable-{maj - 1}.txt"
+            ) as r:
+                stable = r.read().decode().strip()
+                maj_min = _maj_min(stable)
+        else:
+            maj_min = (maj, min - 1)
+    elif snap_version.startswith("latest") or "/" not in snap_version:
+        with urllib.request.urlopen("https://dl.k8s.io/release/stable.txt") as r:
+            stable = r.read().decode().strip()
+            maj_min = _maj_min(stable)
+
+    flavor_track = {"": "classic", "strict": ""}.get(config.FLAVOR, config.FLAVOR)
+    track = f"{maj_min[0]}.{maj_min[1]}" + (flavor_track and f"-{flavor_track}")
+    LOG.info("Previous track for %s is from track: %s", snap_version, track)
+    return track