Merge pull request #1095 from freedomofpress/complete-test-vms-exist

Ensure outdated templates are removed

tests/base.py

@@ -6,7 +6,15 @@
 from qubesadmin import Qubes
 
 # Reusable constant for DRY import across tests
-WANTED_VMS = ["sd-gpg", "sd-log", "sd-proxy", "sd-app", "sd-viewer", "sd-whonix", "sd-devices"]
+DEBIAN_VERSION = "bookworm"
+SD_TEMPLATE_BASE = f"sd-base-{DEBIAN_VERSION}-template"
+SD_TEMPLATE_LARGE = f"sd-large-{DEBIAN_VERSION}-template"
+SD_TEMPLATE_SMALL = f"sd-small-{DEBIAN_VERSION}-template"
+
+SD_VMS = ["sd-gpg", "sd-log", "sd-proxy", "sd-app", "sd-viewer", "sd-whonix", "sd-devices"]
+SD_DVM_TEMPLATES = ["sd-devices-dvm", "sd-proxy-dvm"]
+SD_TEMPLATES = [SD_TEMPLATE_BASE, SD_TEMPLATE_LARGE, SD_TEMPLATE_SMALL]
+
 CURRENT_FEDORA_VERSION = "40"
 CURRENT_FEDORA_TEMPLATE = "fedora-" + CURRENT_FEDORA_VERSION + "-xfce"
 CURRENT_FEDORA_DVM = "fedora-" + CURRENT_FEDORA_VERSION + "-dvm"
@@ -267,14 +275,25 @@ def test_enforced_apparmor_profiles(self):
 class SD_Unnamed_DVM_Local_Test(SD_VM_Local_Test):
     """Tests disposables based on the provided DVM template"""
 
-    def setUp(self, dispvm_template_name):
-        self.app = Qubes()
-        self.vm_name = f"{dispvm_template_name}-disposable"
-
-        if self.vm_name not in self.app.domains:
-            cmd_create_disp = (
-                f"qvm-create --disp --property auto_cleanup=True "
-                f"--template {dispvm_template_name} {self.vm_name}"
-            )
-            subprocess.run(cmd_create_disp.split(), check=True)
-        super().setUp()
+    @classmethod
+    def _kill_test_vm(cls):
+        subprocess.run(["qvm-kill", cls.vm_name], check=True)
+
+    @classmethod
+    def setUpClass(cls, dispvm_template_name):
+        cls.app = Qubes()
+        cls.vm_name = f"{dispvm_template_name}-disposable"
+
+        # VM was running and needs a restart to test on the latest version
+        if cls.vm_name in cls.app.domains:
+            cls._kill_test_vm()
+        # Create disposable based on specified template
+        cmd_create_disp = (
+            f"qvm-create --disp --property auto_cleanup=True "
+            f"--template {dispvm_template_name} {cls.vm_name}"
+        )
+        subprocess.run(cmd_create_disp.split(), check=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._kill_test_vm()

tests/test_viewer.py

@@ -5,8 +5,12 @@
 
 
 class SD_Viewer_Tests(SD_Unnamed_DVM_Local_Test):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass("sd-viewer")
+
     def setUp(self):
-        super().setUp("sd-viewer")
+        super().setUp()
         self.expected_config_keys = {"SD_MIME_HANDLING"}
         # this is not a comprehensive list, just a few that users are likely to use
         self.enforced_apparmor_profiles = {

tests/test_vms_exist.py

@@ -2,11 +2,16 @@
 import subprocess
 import unittest
 
-from base import WANTED_VMS
+from base import (
+    SD_DVM_TEMPLATES,
+    SD_TEMPLATE_BASE,
+    SD_TEMPLATE_LARGE,
+    SD_TEMPLATE_SMALL,
+    SD_TEMPLATES,
+    SD_VMS,
+)
 from qubesadmin import Qubes
 
-DEBIAN_VERSION = "bookworm"
-
 
 class SD_VM_Tests(unittest.TestCase):
     def setUp(self):
@@ -20,17 +25,17 @@ def tearDown(self):
         pass
 
     def test_expected(self):
-        vm_set = set(self.app.domains)
-        for test_vm in WANTED_VMS:
-            self.assertIn(test_vm, vm_set)
+        sdw_tagged_vm_names = [vm.name for vm in self.sdw_tagged_vms]
+        expected_vms = set(SD_VMS + SD_DVM_TEMPLATES + SD_TEMPLATES)
+        self.assertEqual(set(sdw_tagged_vm_names), set(expected_vms))
 
     def test_grsec_kernel(self):
         """
         Confirms expected grsecurity-patched kernel is running.
         """
         # base doesn't have kernel configured and whonix uses dom0 kernel
         # TODO: test in sd-viewer based dispVM
-        exceptions = [f"sd-base-{DEBIAN_VERSION}-template", "sd-whonix", "sd-viewer"]
+        exceptions = [SD_TEMPLATE_BASE, "sd-whonix", "sd-viewer"]
 
         for vm in self.sdw_tagged_vms:
             if vm.name in exceptions:
@@ -97,7 +102,7 @@ def test_sd_proxy_dvm(self):
         vm = self.app.domains["sd-proxy-dvm"]
         self.assertTrue(vm.template_for_dispvms)
         self.assertEqual(vm.netvm.name, "sd-whonix")
-        self.assertEqual(vm.template, f"sd-small-{DEBIAN_VERSION}-template")
+        self.assertEqual(vm.template, SD_TEMPLATE_SMALL)
         self.assertIsNone(vm.default_dispvm)
         self.assertIn("sd-workstation", vm.tags)
         self.assertFalse(vm.autostart)
@@ -108,7 +113,7 @@ def test_sd_app_config(self):
         vm = self.app.domains["sd-app"]
         nvm = vm.netvm
         self.assertIsNone(nvm)
-        self.assertEqual(vm.template, f"sd-small-{DEBIAN_VERSION}-template")
+        self.assertEqual(vm.template, SD_TEMPLATE_SMALL)
         self.assertFalse(vm.provides_network)
         self.assertFalse(vm.template_for_dispvms)
         self.assertNotIn("service.securedrop-log-server", vm.features)
@@ -130,7 +135,7 @@ def test_sd_viewer_config(self):
         vm = self.app.domains["sd-viewer"]
         nvm = vm.netvm
         self.assertIsNone(nvm)
-        self.assertEqual(vm.template, f"sd-large-{DEBIAN_VERSION}-template")
+        self.assertEqual(vm.template, SD_TEMPLATE_LARGE)
         self.assertFalse(vm.provides_network)
         self.assertTrue(vm.template_for_dispvms)
         self.assertIn("sd-workstation", vm.tags)
@@ -144,7 +149,7 @@ def test_sd_gpg_config(self):
         nvm = vm.netvm
         self.assertIsNone(nvm)
         # No sd-gpg-template, since keyring is managed in $HOME
-        self.assertEqual(vm.template, f"sd-small-{DEBIAN_VERSION}-template")
+        self.assertEqual(vm.template, SD_TEMPLATE_SMALL)
         self.assertTrue(vm.autostart)
         self.assertFalse(vm.provides_network)
         self.assertFalse(vm.template_for_dispvms)
@@ -155,7 +160,7 @@ def test_sd_log_config(self):
         vm = self.app.domains["sd-log"]
         nvm = vm.netvm
         self.assertIsNone(nvm)
-        self.assertEqual(vm.template, f"sd-small-{DEBIAN_VERSION}-template")
+        self.assertEqual(vm.template, SD_TEMPLATE_SMALL)
         self.assertTrue(vm.autostart)
         self.assertFalse(vm.provides_network)
         self.assertFalse(vm.template_for_dispvms)
@@ -173,20 +178,20 @@ def test_sd_log_config(self):
         self.assertEqual(vol.size, size * 1024 * 1024 * 1024)
 
     def sd_app_template(self):
-        vm = self.app.domains[f"sd-small-{DEBIAN_VERSION}-template"]
+        vm = self.app.domains[SD_TEMPLATE_SMALL]
         nvm = vm.netvm
         self.assertIsNone(nvm)
         self.assertIn("sd-workstation", vm.tags)
 
     def sd_viewer_template(self):
-        vm = self.app.domains[f"sd-large-{DEBIAN_VERSION}-template"]
+        vm = self.app.domains[SD_TEMPLATE_LARGE]
         nvm = vm.netvm
         self.assertIsNone(nvm)
         self.assertIn("sd-workstation", vm.tags)
         self.assertTrue(vm.template_for_dispvms)
 
     def sd_export_template(self):
-        vm = self.app.domains[f"sd-large-{DEBIAN_VERSION}-template"]
+        vm = self.app.domains[SD_TEMPLATE_LARGE]
         nvm = vm.netvm
         self.assertIsNone(nvm)
         self.assertIn("sd-workstation", vm.tags)
@@ -216,14 +221,14 @@ def sd_export(self):
         self._check_service_running(vm, "securedrop-mime-handling")
 
     def sd_small_template(self):
-        vm = self.app.domains[f"sd-small-{DEBIAN_VERSION}-template"]
+        vm = self.app.domains[SD_TEMPLATE_SMALL]
         nvm = vm.netvm
         self.assertIsNone(nvm)
         self.assertIn("sd-workstation", vm.tags)
         self.assertFalse(vm.template_for_dispvms)
 
     def sd_large_template(self):
-        vm = self.app.domains[f"sd-large-{DEBIAN_VERSION}-template"]
+        vm = self.app.domains[SD_TEMPLATE_LARGE]
         nvm = vm.netvm
         self.assertIsNone(nvm)
         self.assertIn("sd-workstation", vm.tags)

tests/test_vms_platform.py

@@ -3,7 +3,7 @@
 import subprocess
 import unittest
 
-from base import CURRENT_FEDORA_TEMPLATE, WANTED_VMS
+from base import CURRENT_FEDORA_TEMPLATE, SD_VMS
 from qubesadmin import Qubes
 
 BULLSEYE_STRING = "Debian GNU/Linux 11 (bullseye)"
@@ -124,7 +124,7 @@ def test_all_sd_vms_uptodate(self):
         Asserts that all VMs have all available apt packages at the latest
         versions, with no updates pending.
         """
-        for vm_name in WANTED_VMS:
+        for vm_name in SD_VMS:
             vm = self.app.domains[vm_name]
             self._ensure_packages_up_to_date(vm)
 
@@ -162,7 +162,7 @@ def test_all_sd_vm_platforms(self):
         """
         # Would prefer to use a feature like pytest.mark.parametrize
         # for better error output here, but not available in dom0.
-        for vm_name in WANTED_VMS:
+        for vm_name in SD_VMS:
             if vm_name == "sd-viewer":
                 # sd-viewer is unable to start because of the securedrop-mime-handling
                 # systemd service failing, so skip it here.
@@ -188,7 +188,7 @@ def test_all_sd_vm_apt_sources(self):
         Therefore, even if multiple VMs are NOT running a supported platform,
         only a single failure will be reported.
         """
-        for vm_name in WANTED_VMS:
+        for vm_name in SD_VMS:
             if vm_name == "sd-viewer":
                 # sd-viewer is unable to start because of the securedrop-mime-handling
                 # systemd service failing, so skip it here.