Use test-level tags in CI (#850)

Assigns a test level to each integration test and ensures that the integration tests fail if a tag is missing for a test. Also, enables the tags in the CI
canonical · Nov 29, 2024 · eca8749 · eca8749
1 parent 8ee9fc7
commit eca8749
Show file tree

Hide file tree

Showing 25 changed files with 172 additions and 121 deletions.
diff --git a/.github/workflows/integration-informing.yaml b/.github/workflows/integration-informing.yaml
@@ -90,7 +90,7 @@ jobs:
           TEST_FLAVOR: ${{ matrix.patch }}
           TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports
         run: |
-          cd tests/integration && sg lxd -c 'tox -e integration'
+          cd tests/integration && sg lxd -c 'tox -e integration -- --tags pull_request'
       - name: Prepare inspection reports
         if: failure()
         run: |

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -107,7 +107,7 @@ jobs:
           TEST_VERSION_UPGRADE_MIN_RELEASE: "1.31"
           TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]'
         run: |
-          cd tests/integration && sg lxd -c 'tox -e integration'
+          cd tests/integration && sg lxd -c 'tox -e integration -- --tags pull_request'
       - name: Prepare inspection reports
         if: failure()
         run: |

diff --git a/.github/workflows/nightly-test.yaml b/.github/workflows/nightly-test.yaml
@@ -51,7 +51,7 @@ jobs:
           TEST_MIRROR_LIST: '[{"name": "ghcr.io", "port": 5000, "remote": "https://ghcr.io", "username": "${{ github.actor }}", "password": "${{ secrets.GITHUB_TOKEN }}"}, {"name": "docker.io", "port": 5001, "remote": "https://registry-1.docker.io", "username": "", "password": ""}, {"name": "rocks.canonical.com", "port": 5002, "remote": "https://rocks.canonical.com/cdk"}]'
         run: |
           export PATH="/home/runner/.local/bin:$PATH"
-          cd tests/integration && sg lxd -c 'tox -vve integration'
+          cd tests/integration && sg lxd -c 'tox -vve integration -- --tags up_to_nightly '
       - name: Prepare inspection reports
         if: failure()
         run: |

diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
@@ -7,7 +7,7 @@
 from typing import Generator, Iterator, List, Optional, Union
 
 import pytest
-from test_util import config, harness, util
+from test_util import config, harness, tags, util
 from test_util.etcd import EtcdCluster
 from test_util.registry import Registry
 
@@ -20,6 +20,21 @@
 PRELOADED_SNAPS = ["snapd", "core20"]
 
 
+def pytest_itemcollected(item):
+    """
+    A hook to ensure all tests have at least one tag before execution.
+    """
+    # Check for tags in the pytest.mark attributes
+    marked_tags = [mark for mark in item.iter_markers(name="tags")]
+    if not marked_tags or not any(
+        tag.args[0] in tags.TEST_LEVELS for tag in marked_tags
+    ):
+        pytest.fail(
+            f"The test {item.nodeid} does not have one of the test level tags."
+            f"Please add at least one test-level tag using @pytest.mark.tags ({tags.TEST_LEVELS})."
+        )
+
+
 def _harness_clean(h: harness.Harness):
     "Clean up created instances within the test harness."
 

diff --git a/tests/integration/tests/test_annotations.py b/tests/integration/tests/test_annotations.py
@@ -0,0 +1,108 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+from typing import List
+
+import pytest
+from test_util import config, harness, tags, util
+
+LOG = logging.getLogger(__name__)
+
+
+@pytest.mark.node_count(3)
+@pytest.mark.bootstrap_config(
+    (config.MANIFESTS_DIR / "bootstrap-no-k8s-node-remove.yaml").read_text()
+)
+@pytest.mark.tags(tags.NIGHTLY)
+def test_no_remove(instances: List[harness.Instance]):
+    cluster_node = instances[0]
+    joining_cp = instances[1]
+    joining_worker = instances[2]
+
+    join_token = util.get_join_token(cluster_node, joining_cp)
+    join_token_worker = util.get_join_token(cluster_node, joining_worker, "--worker")
+    util.join_cluster(joining_cp, join_token)
+    util.join_cluster(joining_worker, join_token_worker)
+
+    util.wait_until_k8s_ready(cluster_node, instances)
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 3, "nodes should have joined cluster"
+
+    assert "control-plane" in util.get_local_node_status(cluster_node)
+    assert "control-plane" in util.get_local_node_status(joining_cp)
+    assert "worker" in util.get_local_node_status(joining_worker)
+
+    # TODO: k8sd sometimes fails when requested to remove nodes immediately
+    # after bootstrapping the cluster. It seems that it takes a little
+    # longer for trust store changes to be propagated to all nodes, which
+    # should probably be fixed on the microcluster side.
+    #
+    # For now, we'll perform some retries.
+    #
+    #   failed to POST /k8sd/cluster/remove: failed to delete cluster member
+    #   k8s-integration-c1aee0-2: No truststore entry found for node with name
+    #   "k8s-integration-c1aee0-2"
+    util.stubbornly(retries=3, delay_s=5).on(cluster_node).exec(
+        ["k8s", "remove-node", joining_cp.id]
+    )
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 3, "cp node should not have been removed from cluster"
+    cluster_node.exec(["k8s", "remove-node", joining_worker.id])
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 3, "worker node should not have been removed from cluster"
+
+
+@pytest.mark.node_count(3)
+@pytest.mark.bootstrap_config(
+    (config.MANIFESTS_DIR / "bootstrap-skip-service-stop.yaml").read_text()
+)
+@pytest.mark.tags(tags.NIGHTLY)
+def test_skip_services_stop_on_remove(instances: List[harness.Instance]):
+    cluster_node = instances[0]
+    joining_cp = instances[1]
+    worker = instances[2]
+
+    join_token = util.get_join_token(cluster_node, joining_cp)
+    util.join_cluster(joining_cp, join_token)
+
+    join_token_worker = util.get_join_token(cluster_node, worker, "--worker")
+    util.join_cluster(worker, join_token_worker)
+
+    util.wait_until_k8s_ready(cluster_node, instances)
+
+    # TODO: skip retrying this once the microcluster trust store issue is addressed.
+    util.stubbornly(retries=3, delay_s=5).on(cluster_node).exec(
+        ["k8s", "remove-node", joining_cp.id]
+    )
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 2, "cp node should have been removed from the cluster"
+    services = joining_cp.exec(
+        ["snap", "services", "k8s"], capture_output=True, text=True
+    ).stdout.split("\n")[1:-1]
+    for service in services:
+        if "k8s-apiserver-proxy" in service:
+            assert (
+                " inactive " in service
+            ), "apiserver proxy should be inactive on control-plane"
+        else:
+            assert " active " in service, f"'{service}' should be active"
+
+    cluster_node.exec(["k8s", "remove-node", worker.id])
+    nodes = util.ready_nodes(cluster_node)
+    assert len(nodes) == 1, "worker node should have been removed from the cluster"
+    services = worker.exec(
+        ["snap", "services", "k8s"], capture_output=True, text=True
+    ).stdout.split("\n")[1:-1]
+    for service in services:
+        for expected_active_service in [
+            "containerd",
+            "k8sd",
+            "kubelet",
+            "kube-proxy",
+            "k8s-apiserver-proxy",
+        ]:
+            if expected_active_service in service:
+                assert (
+                    " active " in service
+                ), f"{expected_active_service} should be active on worker"
diff --git a/...gration/tests/test_util/test_bootstrap.py → tests/integration/tests/test_bootstrap.py b/...gration/tests/test_util/test_bootstrap.py → tests/integration/tests/test_bootstrap.py
@@ -4,11 +4,12 @@
 from typing import List
 
 import pytest
-from test_util import harness
+from test_util import harness, tags
 
 
 @pytest.mark.node_count(1)
 @pytest.mark.disable_k8s_bootstrapping()
+@pytest.mark.tags(tags.NIGHTLY)
 def test_microk8s_installed(instances: List[harness.Instance]):
     instance = instances[0]
     instance.exec("snap install microk8s --classic".split())

diff --git a/tests/integration/tests/test_cilium_e2e.py b/tests/integration/tests/test_cilium_e2e.py
@@ -7,7 +7,7 @@
 from typing import List
 
 import pytest
-from test_util import config, harness, util
+from test_util import config, harness, tags, util
 
 LOG = logging.getLogger(__name__)
 
@@ -24,6 +24,7 @@
     os.getenv("TEST_CILIUM_E2E") in ["false", None],
     reason="Test is known to be flaky on GitHub Actions",
 )
+@pytest.mark.tags(tags.WEEKLY)
 def test_cilium_e2e(instances: List[harness.Instance]):
     instance = instances[0]
     instance.exec(["bash", "-c", "mkdir -p ~/.kube"])

diff --git a/tests/integration/tests/test_cleanup.py b/tests/integration/tests/test_cleanup.py
@@ -5,7 +5,7 @@
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import harness, tags, util
 
 LOG = logging.getLogger(__name__)
 
@@ -18,6 +18,7 @@
 
 
 @pytest.mark.node_count(1)
+@pytest.mark.tags(tags.NIGHTLY)
 def test_node_cleanup(instances: List[harness.Instance], tmp_path):
     instance = instances[0]
     util.wait_for_dns(instance)

diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py
@@ -11,12 +11,13 @@
 import pytest
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
-from test_util import config, harness, util
+from test_util import config, harness, tags, util
 
 LOG = logging.getLogger(__name__)
 
 
 @pytest.mark.node_count(2)
+@pytest.mark.tags(tags.PULL_REQUEST)
 def test_control_plane_nodes(instances: List[harness.Instance]):
     cluster_node = instances[0]
     joining_node = instances[1]
@@ -41,6 +42,7 @@ def test_control_plane_nodes(instances: List[harness.Instance]):
 
 @pytest.mark.node_count(2)
 @pytest.mark.snap_versions([util.previous_track(config.SNAP), config.SNAP])
+@pytest.mark.tags(tags.NIGHTLY)
 def test_mixed_version_join(instances: List[harness.Instance]):
     """Test n versioned node joining a n-1 versioned cluster."""
     cluster_node = instances[0]  # bootstrapped on the previous channel
@@ -65,6 +67,7 @@ def test_mixed_version_join(instances: List[harness.Instance]):
 
 
 @pytest.mark.node_count(3)
+@pytest.mark.tags(tags.PULL_REQUEST)
 def test_worker_nodes(instances: List[harness.Instance]):
     cluster_node = instances[0]
     joining_node = instances[1]
@@ -98,104 +101,7 @@ def test_worker_nodes(instances: List[harness.Instance]):
 
 
 @pytest.mark.node_count(3)
-@pytest.mark.bootstrap_config(
-    (config.MANIFESTS_DIR / "bootstrap-no-k8s-node-remove.yaml").read_text()
-)
-def test_no_remove(instances: List[harness.Instance]):
-    cluster_node = instances[0]
-    joining_cp = instances[1]
-    joining_worker = instances[2]
-
-    join_token = util.get_join_token(cluster_node, joining_cp)
-    join_token_worker = util.get_join_token(cluster_node, joining_worker, "--worker")
-    util.join_cluster(joining_cp, join_token)
-    util.join_cluster(joining_worker, join_token_worker)
-
-    util.wait_until_k8s_ready(cluster_node, instances)
-    nodes = util.ready_nodes(cluster_node)
-    assert len(nodes) == 3, "nodes should have joined cluster"
-
-    assert "control-plane" in util.get_local_node_status(cluster_node)
-    assert "control-plane" in util.get_local_node_status(joining_cp)
-    assert "worker" in util.get_local_node_status(joining_worker)
-
-    # TODO: k8sd sometimes fails when requested to remove nodes immediately
-    # after bootstrapping the cluster. It seems that it takes a little
-    # longer for trust store changes to be propagated to all nodes, which
-    # should probably be fixed on the microcluster side.
-    #
-    # For now, we'll perform some retries.
-    #
-    #   failed to POST /k8sd/cluster/remove: failed to delete cluster member
-    #   k8s-integration-c1aee0-2: No truststore entry found for node with name
-    #   "k8s-integration-c1aee0-2"
-    util.stubbornly(retries=3, delay_s=5).on(cluster_node).exec(
-        ["k8s", "remove-node", joining_cp.id]
-    )
-    nodes = util.ready_nodes(cluster_node)
-    assert len(nodes) == 3, "cp node should not have been removed from cluster"
-    cluster_node.exec(["k8s", "remove-node", joining_worker.id])
-    nodes = util.ready_nodes(cluster_node)
-    assert len(nodes) == 3, "worker node should not have been removed from cluster"
-
-
-@pytest.mark.node_count(3)
-@pytest.mark.bootstrap_config(
-    (config.MANIFESTS_DIR / "bootstrap-skip-service-stop.yaml").read_text()
-)
-def test_skip_services_stop_on_remove(instances: List[harness.Instance]):
-    cluster_node = instances[0]
-    joining_cp = instances[1]
-    worker = instances[2]
-
-    join_token = util.get_join_token(cluster_node, joining_cp)
-    util.join_cluster(joining_cp, join_token)
-
-    join_token_worker = util.get_join_token(cluster_node, worker, "--worker")
-    util.join_cluster(worker, join_token_worker)
-
-    util.wait_until_k8s_ready(cluster_node, instances)
-
-    # TODO: skip retrying this once the microcluster trust store issue is addressed.
-    util.stubbornly(retries=3, delay_s=5).on(cluster_node).exec(
-        ["k8s", "remove-node", joining_cp.id]
-    )
-    nodes = util.ready_nodes(cluster_node)
-    assert len(nodes) == 2, "cp node should have been removed from the cluster"
-    services = joining_cp.exec(
-        ["snap", "services", "k8s"], capture_output=True, text=True
-    ).stdout.split("\n")[1:-1]
-    print(services)
-    for service in services:
-        if "k8s-apiserver-proxy" in service:
-            assert (
-                " inactive " in service
-            ), "apiserver proxy should be inactive on control-plane"
-        else:
-            assert " active " in service, "service should be active"
-
-    cluster_node.exec(["k8s", "remove-node", worker.id])
-    nodes = util.ready_nodes(cluster_node)
-    assert len(nodes) == 1, "worker node should have been removed from the cluster"
-    services = worker.exec(
-        ["snap", "services", "k8s"], capture_output=True, text=True
-    ).stdout.split("\n")[1:-1]
-    print(services)
-    for service in services:
-        for expected_active_service in [
-            "containerd",
-            "k8sd",
-            "kubelet",
-            "kube-proxy",
-            "k8s-apiserver-proxy",
-        ]:
-            if expected_active_service in service:
-                assert (
-                    " active " in service
-                ), f"{expected_active_service} should be active on worker"
-
-
-@pytest.mark.node_count(3)
+@pytest.mark.tags(tags.NIGHTLY)
 def test_join_with_custom_token_name(instances: List[harness.Instance]):
     cluster_node = instances[0]
     joining_cp = instances[1]
@@ -254,6 +160,7 @@ def test_join_with_custom_token_name(instances: List[harness.Instance]):
 @pytest.mark.bootstrap_config(
     (config.MANIFESTS_DIR / "bootstrap-csr-auto-approve.yaml").read_text()
 )
+@pytest.mark.tags(tags.NIGHTLY)
 def test_cert_refresh(instances: List[harness.Instance]):
     cluster_node = instances[0]
     joining_worker = instances[1]

diff --git a/tests/integration/tests/test_clustering_race.py b/tests/integration/tests/test_clustering_race.py
@@ -4,10 +4,11 @@
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import harness, tags, util
 
 
 @pytest.mark.node_count(3)
+@pytest.mark.tags(tags.NIGHTLY)
 def test_wrong_token_race(instances: List[harness.Instance]):
     cluster_node = instances[0]
 

diff --git a/tests/integration/tests/test_config_propagation.py b/tests/integration/tests/test_config_propagation.py
@@ -5,12 +5,13 @@
 from typing import List
 
 import pytest
-from test_util import harness, util
+from test_util import harness, tags, util
 
 LOG = logging.getLogger(__name__)
 
 
 @pytest.mark.node_count(3)
+@pytest.mark.tags(tags.NIGHTLY)
 def test_config_propagation(instances: List[harness.Instance]):
     initial_node = instances[0]
     joining_cplane_node = instances[1]