Composition tests: Make test_automatic_host_removal more robust

Forces the Check_MK services to be in CRIT state

This makes the test independent from core scheduling and the whole check
cycle which shall not be tested here. Instead of relying on it, we
disable the core scheduling and set the services to CRIT state. This way
we can be sure that the conditions for the host removal are met.

Change-Id: I6df27ef5003d4e01ecae26923eb184e678fef120

cmk/gui/watolib/automatic_host_removal.py

@@ -89,7 +89,8 @@ def execute_host_removal_job() -> None:
 
 
 def _init_logging() -> None:
-    handler = FileHandler(Path(log_dir, "automatic-host-removal.log"), encoding="utf-8")
+    handler = FileHandler(log_file := Path(log_dir, "automatic-host-removal.log"), encoding="utf-8")
+    _LOGGER.info("Logging host removal to %s", log_file)
     handler.setFormatter(Formatter("%(asctime)s [%(levelno)s] [%(name)s %(process)d] %(message)s"))
     del _LOGGER.handlers[:]  # Remove all previously existing handlers
     _LOGGER.addHandler(handler)

tests/composition/test_automatic_host_removal.py

@@ -5,6 +5,8 @@
 
 import logging
 import os
+from collections.abc import Iterator
+from contextlib import contextmanager
 
 import pytest
 
@@ -29,52 +31,73 @@ def test_automatic_host_removal(
     unresolvable_host_central = "not-dns-resolvable-central"
     unresolvable_host_remote = "not-dns-resolvable-remote"
 
-    central_site.openapi.hosts.create(
-        hostname=unresolvable_host_central,
-        attributes={"site": central_site.id},
-    )
-    central_site.openapi.hosts.create(
-        hostname=unresolvable_host_remote,
-        attributes={"site": remote_site.id},
-    )
-
-    rule_id = central_site.openapi.rules.create(
-        ruleset_name="automatic_host_removal",
-        value=("enabled", {"checkmk_service_crit": 1}),
-        conditions={
-            "host_name": {
-                "match_on": [unresolvable_host_remote, unresolvable_host_central],
-                "operator": "one_of",
-            }
-        },
-    )
-
-    central_site.openapi.changes.activate_and_wait_for_completion(force_foreign_changes=True)
+    with disable_core_check_scheduling(central_site), disable_core_check_scheduling(remote_site):
+        central_site.openapi.hosts.create(
+            hostname=unresolvable_host_central,
+            attributes={"site": central_site.id},
+        )
+        central_site.openapi.hosts.create(
+            hostname=unresolvable_host_remote,
+            attributes={"site": remote_site.id},
+        )
 
-    try:
+        rule_id = central_site.openapi.rules.create(
+            ruleset_name="automatic_host_removal",
+            value=("enabled", {"checkmk_service_crit": 1}),
+            conditions={
+                "host_name": {
+                    "match_on": [unresolvable_host_remote, unresolvable_host_central],
+                    "operator": "one_of",
+                }
+            },
+        )
 
-        def _host_removal_done() -> bool:
-            hostnames = set(central_site.openapi.hosts.get_all_names())
-            return not hostnames.intersection({unresolvable_host_central, unresolvable_host_remote})
+        central_site.openapi.changes.activate_and_wait_for_completion(force_foreign_changes=True)
 
-        logger.info("Waiting for hosts to be removed")
-        wait_until(
-            _host_removal_done,
-            timeout=180,
-            interval=20,
+        central_site.send_service_check_result(
+            unresolvable_host_central, "Check_MK", 2, "FAKE CRIT"
         )
+        remote_site.send_service_check_result(unresolvable_host_remote, "Check_MK", 2, "FAKE CRIT")
 
-        logger.info("Waiting for changes to be activated")
-        wait_until(
-            lambda: not central_site.openapi.changes.get_pending(),
-            timeout=180,
-            interval=20,
-        )
+        try:
+
+            def _host_removal_done() -> bool:
+                hostnames = set(central_site.openapi.hosts.get_all_names())
+                return not hostnames.intersection(
+                    {unresolvable_host_central, unresolvable_host_remote}
+                )
 
+            logger.info("Waiting for hosts to be removed")
+            wait_until(
+                _host_removal_done,
+                timeout=180,
+                interval=20,
+            )
+
+            logger.info("Waiting for changes to be activated")
+            wait_until(
+                lambda: not central_site.openapi.changes.get_pending(),
+                timeout=180,
+                interval=20,
+            )
+
+        finally:
+            central_site.openapi.rules.delete(rule_id=rule_id)
+            if unresolvable_host_central in central_site.openapi.hosts.get_all_names():
+                central_site.openapi.hosts.delete(unresolvable_host_central)
+            if unresolvable_host_remote in central_site.openapi.hosts.get_all_names():
+                central_site.openapi.hosts.delete(unresolvable_host_remote)
+            central_site.openapi.changes.activate_and_wait_for_completion(
+                force_foreign_changes=True
+            )
+
+
+@contextmanager
+def disable_core_check_scheduling(site: Site) -> Iterator[None]:
+    site.stop_host_checks()
+    site.stop_active_services()
+    try:
+        yield
     finally:
-        central_site.openapi.rules.delete(rule_id=rule_id)
-        if unresolvable_host_central in central_site.openapi.hosts.get_all_names():
-            central_site.openapi.hosts.delete(unresolvable_host_central)
-        if unresolvable_host_remote in central_site.openapi.hosts.get_all_names():
-            central_site.openapi.hosts.delete(unresolvable_host_remote)
-        central_site.openapi.changes.activate_and_wait_for_completion(force_foreign_changes=True)
+        site.start_host_checks()
+        site.start_active_services()

tests/testlib/utils.py

@@ -520,7 +520,7 @@ def wait_until(condition: Callable[[], bool], timeout: float = 1, interval: floa
     logger.info("Waiting for %r to finish for %ds", condition, timeout)
     while time.time() - start < timeout:
         if condition():
-            logger.info("Wait for %r finished after %ss", condition, time.time() - start)
+            logger.info("Wait for %r finished after %0.2fs", condition, time.time() - start)
             return  # Success. Stop waiting...
         time.sleep(interval)