Test-update from backup with demo-site

Change-Id: I3ef8dd660d66fbb0e08e9620bb224f2b87652637

tests/testlib/site.py

@@ -1316,7 +1316,9 @@ def __init__(
     def sites(self) -> Mapping[str, Site]:
         return self._sites
 
-    def get_site(self, name: str, init_livestatus: bool = True) -> Site:
+    def get_site(
+        self, name: str, init_livestatus: bool = True, activate_changes: bool = True
+    ) -> Site:
         site = self._site_obj(name)
 
         site.create()
@@ -1330,8 +1332,9 @@ def get_site(self, name: str, init_livestatus: bool = True) -> Site:
         site.start()
         site.prepare_for_tests()
 
-        # There seem to be still some changes that want to be activated
-        site.activate_changes_and_wait_for_core_reload()
+        if activate_changes:
+            # There seem to be still some changes that want to be activated
+            site.activate_changes_and_wait_for_core_reload()
 
         logger.debug("Created site %s", site.id)
         return site

tests/update/conftest.py

@@ -38,6 +38,12 @@ def pytest_addoption(parser):
         default=False,
         help="Use the latest base-version only.",
     )
+    parser.addoption(
+        "--store-lost-services",
+        action="store_true",
+        default=False,
+        help="Store list of lost services in a json reference.",
+    )
 
 
 @dataclasses.dataclass

tests/update/lost_services_demo.json

@@ -0,0 +1,127 @@
+{
+    "aws-web": [],
+    "play.checkmk.com": [],
+    "firewall.company.com": [],
+    "hp.ilo4.company.com": [],
+    "printer-1.company.com": [],
+    "switch-1.company.com": [],
+    "switch-2.company.com": [
+        "Memory",
+        "CPU utilization"
+    ],
+    "srv001.vms.company.com": [],
+    "srv002.vms.company.com": [],
+    "srv004.vms.company.com": [],
+    "srv005.vms.company.com": [],
+    "srv006.vms.company.com": [],
+    "srv007.vms.company.com": [],
+    "srv008.vms.company.com": [],
+    "srv009.vms.company.com": [],
+    "srv013.vms.company.com": [],
+    "webserver.company.com": [],
+    "srv023.company.com": [],
+    "srv023.vms.company.com": [],
+    "srv024.company.com": [],
+    "srv024.vms.company.com": [],
+    "srv035.company.com": [],
+    "srv035.vms.company.com": [],
+    "srv036.company.com": [],
+    "srv036.vms.company.com": [],
+    "srv037.company.com": [],
+    "srv037.vms.company.com": [],
+    "srv050.company.com": [],
+    "srv050.vms.company.com": [],
+    "srv051.company.com": [],
+    "srv051.vms.company.com": [],
+    "windows-server.company.com": [],
+    "app-1.erp.company.com": [
+        "Nginx 10.0.10.10:80 Status"
+    ],
+    "app-2.erp.company.com": [
+        "Nginx 10.0.10.10:80 Status"
+    ],
+    "exchange-1.company.com": [],
+    "exchange-2.company.com": [],
+    "exchange.company.com": [],
+    "ldap-1.company.com": [
+        "Postfix status",
+        "Postfix Queue"
+    ],
+    "ldap-2.company.com": [
+        "Postfix status",
+        "Postfix Queue"
+    ],
+    "loadbalancer1.company.com": [],
+    "loadbalancer2.company.com": [],
+    "mysql-1.erp.company.com": [
+        "Postfix Queue"
+    ],
+    "mysql-2.erp.company.com": [
+        "Postfix Queue"
+    ],
+    "vcenter.company.com": [],
+    "kubernetes.company.com": [],
+    "daemonset_eks_checkmk-monitoring_checkmk-node-collector-container-metrics": [],
+    "daemonset_eks_checkmk-monitoring_checkmk-node-collector-machine-sections": [],
+    "daemonset_eks_kube-system_aws-node": [],
+    "daemonset_eks_kube-system_ebs-csi-node": [],
+    "daemonset_eks_kube-system_ebs-csi-node-windows": [],
+    "daemonset_eks_kube-system_kube-proxy": [],
+    "deployment_eks_checkmk-monitoring_checkmk-cluster-collector": [],
+    "deployment_eks_ecommerce-services_authentication-service": [],
+    "deployment_eks_ecommerce-services_payment-service": [],
+    "deployment_eks_ecommerce-services_product-catalog-service": [],
+    "deployment_eks_ecommerce-services_store-cart-service": [],
+    "deployment_eks_kube-system_coredns": [],
+    "deployment_eks_kube-system_ebs-csi-controller": [],
+    "namespace_eks_checkmk-monitoring": [],
+    "namespace_eks_ecommerce-services": [],
+    "namespace_eks_kube-system": [],
+    "node_eks_i-0c8dca59e0fc7ef61.eu-central-1.compute.internal": [],
+    "node_eks_i-0d7776d349d81f15c.eu-central-1.compute.internal": [],
+    "pod_eks_checkmk-monitoring_checkmk-cluster-collector-547ccd6794-jxzvz": [],
+    "pod_eks_checkmk-monitoring_checkmk-node-collector-container-metrics-gt2x5": [],
+    "pod_eks_checkmk-monitoring_checkmk-node-collector-container-metrics-m6lvb": [],
+    "pod_eks_checkmk-monitoring_checkmk-node-collector-machine-sections-bgdmd": [],
+    "pod_eks_checkmk-monitoring_checkmk-node-collector-machine-sections-kzzdd": [],
+    "pod_eks_ecommerce-services_authentication-service-6dbbf7d965-5bn47": [],
+    "pod_eks_ecommerce-services_authentication-service-6dbbf7d965-6b7s7": [],
+    "pod_eks_ecommerce-services_authentication-service-6dbbf7d965-zhv5x": [],
+    "pod_eks_ecommerce-services_payment-service-69d559b7f-2vbsv": [],
+    "pod_eks_ecommerce-services_payment-service-69d559b7f-6qlml": [],
+    "pod_eks_ecommerce-services_payment-service-69d559b7f-9xqzv": [],
+    "pod_eks_ecommerce-services_payment-service-69d559b7f-hwdbj": [],
+    "pod_eks_ecommerce-services_payment-service-69d559b7f-llqz4": [],
+    "pod_eks_ecommerce-services_product-catalog-service-857bd858cf-b2gqn": [],
+    "pod_eks_ecommerce-services_product-catalog-service-857bd858cf-sl8q8": [],
+    "pod_eks_ecommerce-services_product-catalog-service-857bd858cf-tsmfc": [],
+    "pod_eks_ecommerce-services_product-catalog-service-857bd858cf-wjdr7": [],
+    "pod_eks_ecommerce-services_store-cart-service-54fbd75fd5-82lhx": [],
+    "pod_eks_ecommerce-services_store-cart-service-54fbd75fd5-fd7tl": [],
+    "pod_eks_ecommerce-services_store-cart-service-54fbd75fd5-j4pcs": [],
+    "pod_eks_ecommerce-services_store-cart-service-54fbd75fd5-qqh7t": [],
+    "pod_eks_ecommerce-services_store-cart-service-54fbd75fd5-vkslt": [],
+    "pod_eks_kube-system_aws-node-gsj2f": [],
+    "pod_eks_kube-system_aws-node-svjls": [],
+    "pod_eks_kube-system_coredns-7bc655f56f-p6ph2": [],
+    "pod_eks_kube-system_coredns-7bc655f56f-pbxg2": [],
+    "pod_eks_kube-system_ebs-csi-controller-6b45f6d576-jx67t": [],
+    "pod_eks_kube-system_ebs-csi-controller-6b45f6d576-r47j5": [],
+    "pod_eks_kube-system_ebs-csi-node-kh7fp": [],
+    "pod_eks_kube-system_ebs-csi-node-z9khz": [],
+    "pod_eks_kube-system_kube-proxy-7drhq": [],
+    "pod_eks_kube-system_kube-proxy-bpr2x": [],
+    "15.0.0.207-eu-central-1-i-0c8dca59e0fc7ef61": [
+        "PING"
+    ],
+    "15.0.11.75-eu-central-1-i-0d7776d349d81f15c": [
+        "PING"
+    ],
+    "15.0.3.186-eu-central-1-i-04998884db9ab59be": [
+        "PING"
+    ],
+    "checkmk.com": [],
+    "docs.checkmk.com": [],
+    "exchange.checkmk.com": [],
+    "forum.checkmk.com": []
+}

tests/update/test_update_from_backup.py

@@ -3,6 +3,7 @@
 from collections.abc import Iterator
 from pathlib import Path
 
+import json
 import pytest
 
 from tests.testlib.agent import (
@@ -53,6 +54,19 @@ def _agent_ctl(installed_agent_ctl_in_unknown_state: Path) -> Iterator[Path]:
         yield installed_agent_ctl_in_unknown_state
 
 
+@pytest.fixture(name="site_factory_demo", scope="function")
+def _site_factory_demo():
+    base_version = CMKVersion("2.2.0p4", Edition.CCE, current_base_branch_name())
+    return SiteFactory(version=base_version, prefix="")
+
+
+@pytest.fixture(name="base_site_demo", scope="function")
+def _base_site_demo(site_factory_demo):
+    # Note: to access the UI of the "play" site go to http://localhost/play/check_mk/login.py?_admin
+    site_name = "play"
+    yield from site_factory_demo.get_test_site(site_name, save_results=False)
+
+
 @pytest.mark.skipif(
     os.environ.get("DISTRO") in ("sles-15sp4", "sles-15sp5"),
     reason="Test currently failing for missing `php7`. "
@@ -61,7 +75,7 @@ def _agent_ctl(installed_agent_ctl_in_unknown_state: Path) -> Iterator[Path]:
 def test_update_from_backup(site_factory: SiteFactory, base_site: Site, agent_ctl: Path) -> None:
     backup_path = qa_test_data_path() / Path("update/backups/update_central_backup.tar.gz")
     assert backup_path.exists()
-    
+
     base_site = site_factory.restore_site_from_backup(backup_path, base_site.id, reuse=True)
     hostnames = [_.get("id") for _ in base_site.openapi.get_hosts()]
 
@@ -120,3 +134,95 @@ def test_update_from_backup(site_factory: SiteFactory, base_site: Site, agent_ct
             f"{not_ok_services}"
         )
         assert set(base_ok_services[hostname]).issubset(set(target_ok_services[hostname])), err_msg
+
+
+def test_update_from_backup_demo(
+    site_factory_demo: SiteFactory, base_site_demo: Site, request: pytest.FixtureRequest
+) -> None:
+    store_lost_services = request.config.getoption(name="--store-lost-services")
+    lost_services_path = Path(__file__).parent.resolve() / Path("lost_services_demo.json")
+
+    # MKPs broken: disabled in the demo site via: 'mkp disable play_checkmk 0.0.1' TODO: investigate
+    backup_path = qa_test_data_path() / Path("update/backups/play.checkmk.com.tar.gz")
+    assert backup_path.exists()
+
+    base_site = site_factory_demo.restore_site_from_backup(
+        backup_path, base_site_demo.id, reuse=True
+    )
+
+    base_hostnames = [_.get("id") for _ in base_site.openapi.get_hosts()]
+
+    base_services = {}
+    base_ok_services = {}
+    for hostname in base_hostnames:
+        base_services[hostname] = base_site.get_host_services(hostname)
+        base_ok_services[hostname] = get_services_with_status(base_services[hostname], 0)
+
+        assert len(base_services[hostname]) > 0, f"No services found in host {hostname}"
+
+    target_version = CMKVersion(CMKVersion.DAILY, Edition.CCE, current_base_branch_name())
+
+    site_factory_demo = SiteFactory(
+        version=target_version,
+        prefix="",
+        update_from_git=False,
+        update=True,
+        update_conflict_mode="keepold",
+        enforce_english_gui=False,
+    )
+
+    base_site.stop()
+    target_site = site_factory_demo.get_site(  # perform update via CLI
+        base_site.id, init_livestatus=False, activate_changes=False
+    )
+    target_site.openapi.activate_changes_and_wait_for_completion()
+
+    assert target_site.is_running()
+    assert target_site.version.version == target_version.version, "Version mismatch during update!"
+
+    target_hostnames = [_.get("id") for _ in base_site.openapi.get_hosts()]
+
+    target_services = {}
+    current_lost_services = {}
+    missed_services = {}
+
+    with open(lost_services_path, "r") as json_file:
+        known_lost_services = json.load(json_file)
+
+    for hostname in target_hostnames:
+        target_services[hostname] = target_site.get_host_services(hostname)
+
+        current_lost_services[hostname] = [
+            service
+            for service in base_services[hostname]
+            if service not in target_services[hostname]
+        ]
+
+        if current_lost_services[hostname]:
+            logger.warning(
+                "In the %s host the following services were found in base-version "
+                "but not in target-version: %s",
+                hostname,
+                current_lost_services[hostname],
+            )
+
+        if store_lost_services:
+            # skip assertion if flag given
+            continue
+
+        missed_services[hostname] = [
+            service
+            for service in current_lost_services[hostname]
+            if service not in known_lost_services[hostname]
+        ]
+
+        err_msg = (
+            f"In the {hostname} host the following services were not expected to be missing "
+            f"{missed_services[hostname]}"
+        )
+        assert not missed_services[hostname], err_msg
+
+    if store_lost_services:
+        logger.info("Storing lost services as JSON reference...")
+        with open(lost_services_path, "w") as file:
+            json.dump(current_lost_services, file, indent=4)