14237 FIX Fixed performance problems for various command line operations

SUP-20567

With larger setups (>10000 hosts), loading the host configuration took considerably longer ( **O(n^2)** )  than normal

Change-Id: Idbc1f137f2ec17e45e281e357991253d0f24b258

.werks/14237.md

@@ -0,0 +1,15 @@
+[//]: # (werk v2)
+# Fixed performance problems for various command line operations
+
+key        | value
+---------- | ---
+date       | 2024-09-27T10:40:03+00:00
+version    | 2.3.0p18
+class      | fix
+edition    | cre
+component  | checks
+level      | 1
+compatible | yes
+
+With larger setups (>10000 hosts), loading the host configuration took considerably longer ( **O(n^2)** ) than normal.
+The ‘Parameters for this service’ page, for example, requires the complete host configuration.

cmk/base/config.py

@@ -40,18 +40,13 @@
 import cmk.utils.check_utils
 import cmk.utils.cleanup
 import cmk.utils.config_path
-import cmk.utils.config_warnings as config_warnings
 import cmk.utils.debug
-import cmk.utils.password_store as password_store
 import cmk.utils.paths
-import cmk.utils.piggyback as piggyback
-import cmk.utils.rulesets.ruleset_matcher as ruleset_matcher
-import cmk.utils.rulesets.tuple_rulesets as tuple_rulesets
-import cmk.utils.store as store
 import cmk.utils.store.host_storage
 import cmk.utils.tags
 import cmk.utils.translations
 import cmk.utils.version as cmk_version
+from cmk.utils import config_warnings, password_store, piggyback, store
 from cmk.utils.agent_registration import (
     connection_mode_from_host_config,
     HostAgentConnectionMode,
@@ -80,7 +75,7 @@
 from cmk.utils.log import console
 from cmk.utils.macros import replace_macros_in_str
 from cmk.utils.regex import regex
-from cmk.utils.rulesets import RuleSetName
+from cmk.utils.rulesets import ruleset_matcher, RuleSetName, tuple_rulesets
 from cmk.utils.rulesets.ruleset_matcher import (
     LabelManager,
     LabelSources,
@@ -152,8 +147,7 @@
 )
 
 import cmk.base.api.agent_based.register as agent_based_register
-import cmk.base.default_config as default_config
-import cmk.base.ip_lookup as ip_lookup
+from cmk.base import default_config, ip_lookup
 from cmk.base.api.agent_based.cluster_mode import ClusterMode
 from cmk.base.api.agent_based.plugin_classes import SNMPSectionPlugin
 from cmk.base.api.agent_based.register.check_plugins_legacy import (
@@ -2022,13 +2016,11 @@ def initialize(self) -> ConfigCache:
             ),
             clusters_of=self._clusters_of_cache,
             nodes_of=self._nodes_of_cache,
-            all_configured_hosts=list(
-                set(
-                    itertools.chain(
-                        self.hosts_config.hosts,
-                        self.hosts_config.clusters,
-                        self.hosts_config.shadow_hosts,
-                    )
+            all_configured_hosts=frozenset(
+                itertools.chain(
+                    self.hosts_config.hosts,
+                    self.hosts_config.clusters,
+                    self.hosts_config.shadow_hosts,
                 )
             ),
         )

cmk/utils/rulesets/ruleset_matcher.py

@@ -8,7 +8,7 @@
 import dataclasses
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
 from re import Pattern
-from typing import Any, cast, Generic, NamedTuple, NotRequired, TypeAlias, TypeVar
+from typing import Any, cast, FrozenSet, Generic, NamedTuple, NotRequired, Set, TypeAlias, TypeVar
 
 from typing_extensions import TypedDict
 
@@ -150,7 +150,7 @@ def __init__(
         host_tags: TagsOfHosts,
         host_paths: Mapping[HostName, str],
         label_manager: LabelManager,
-        all_configured_hosts: Sequence[HostName],
+        all_configured_hosts: FrozenSet[HostName],
         clusters_of: Mapping[HostName, Sequence[HostName]],
         nodes_of: Mapping[HostName, Sequence[HostName]],
     ) -> None:
@@ -400,7 +400,7 @@ def __init__(
         host_tags: TagsOfHosts,
         host_paths: Mapping[HostName, str],
         label_manager: LabelManager,
-        all_configured_hosts: Sequence[HostName],
+        all_configured_hosts: FrozenSet[HostName],
         clusters_of: Mapping[HostName, Sequence[HostName]],
         nodes_of: Mapping[HostName, Sequence[HostName]],
     ) -> None:
@@ -450,11 +450,11 @@ def clear_caches(self) -> None:
         self.__host_ruleset_cache.clear()
         self._all_matching_hosts_match_cache.clear()
 
-    def all_processed_hosts(self) -> Sequence[HostName]:
+    def all_processed_hosts(self) -> FrozenSet[HostName]:
         """Returns a set of all processed hosts"""
         return self._all_processed_hosts
 
-    def set_all_processed_hosts(self, all_processed_hosts: Iterable[HostName]) -> None:
+    def set_all_processed_hosts(self, all_processed_hosts: Set[HostName]) -> None:
         involved_clusters: set[HostName] = set()
         involved_nodes: set[HostName] = set()
         for hostname in self._all_processed_hosts:
@@ -465,11 +465,11 @@ def set_all_processed_hosts(self, all_processed_hosts: Iterable[HostName]) -> No
         for hostname in involved_clusters:
             involved_nodes.update(self._nodes_of.get(hostname, []))
 
-        nodes_and_clusters = involved_clusters | involved_nodes | set(all_processed_hosts)
+        nodes_and_clusters = involved_clusters | involved_nodes | all_processed_hosts
 
         # Only add references to configured hosts
         nodes_and_clusters.intersection_update(self._all_configured_hosts)
-        self._all_processed_hosts = list(nodes_and_clusters)
+        self._all_processed_hosts = frozenset(nodes_and_clusters)
 
         # The folder host lookup includes a list of all -processed- hosts within a given
         # folder. Any update with set_all_processed hosts invalidates this cache, because

tests/unit/cmk/utils/rulesets/test_ruleset_matcher.py

@@ -146,7 +146,7 @@ def test_ruleset_matcher_get_host_values_labels(
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[HostName("host1"), HostName("host2")],
+        all_configured_hosts=frozenset([HostName("host1"), HostName("host2")]),
         clusters_of={},
         nodes_of={},
     )
@@ -183,7 +183,7 @@ def test_labels_of_service(monkeypatch: MonkeyPatch) -> None:
             ],
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[test_host, xyz_host],
+        all_configured_hosts=frozenset([test_host, xyz_host]),
         clusters_of={},
         nodes_of={},
     )
@@ -215,7 +215,7 @@ def test_labels_of_service_discovered_labels() -> None:
                 lambda host_name, *args, **kw: {"äzzzz": "eeeeez"} if host_name == test_host else {}
             ),
         ),
-        all_configured_hosts=[test_host],
+        all_configured_hosts=frozenset([test_host]),
         clusters_of={},
         nodes_of={},
     )
@@ -246,12 +246,14 @@ def test_basic_get_host_values() -> None:
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[
-            HostName("abc"),
-            HostName("xyz"),
-            HostName("host1"),
-            HostName("host2"),
-        ],
+        all_configured_hosts=frozenset(
+            [
+                HostName("abc"),
+                HostName("xyz"),
+                HostName("host1"),
+                HostName("host2"),
+            ]
+        ),
         clusters_of={},
         nodes_of={},
     )
@@ -285,13 +287,15 @@ def test_basic_get_host_values_subfolders() -> None:
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[
-            HostName("abc"),
-            HostName("xyz"),
-            HostName("lvl1"),
-            HostName("lvl2"),
-            HostName("lvl1a"),
-        ],
+        all_configured_hosts=frozenset(
+            [
+                HostName("abc"),
+                HostName("xyz"),
+                HostName("lvl1"),
+                HostName("lvl2"),
+                HostName("lvl1a"),
+            ]
+        ),
         clusters_of={},
         nodes_of={},
     )
@@ -361,12 +365,14 @@ def test_basic_host_ruleset_get_merged_dict_values() -> None:
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[
-            HostName("abc"),
-            HostName("xyz"),
-            HostName("host1"),
-            HostName("host2"),
-        ],
+        all_configured_hosts=frozenset(
+            [
+                HostName("abc"),
+                HostName("xyz"),
+                HostName("host1"),
+                HostName("host2"),
+            ]
+        ),
         clusters_of={},
         nodes_of={},
     )
@@ -436,12 +442,14 @@ def test_basic_host_ruleset_get_host_bool_value() -> None:
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[
-            HostName("abc"),
-            HostName("xyz"),
-            HostName("host1"),
-            HostName("host2"),
-        ],
+        all_configured_hosts=frozenset(
+            [
+                HostName("abc"),
+                HostName("xyz"),
+                HostName("host1"),
+                HostName("host2"),
+            ]
+        ),
         clusters_of={},
         nodes_of={},
     )
@@ -559,11 +567,13 @@ def test_ruleset_matcher_get_host_values_tags(
             service_label_rules=(),
             discovered_labels_of_service=lambda *args, **kw: {},
         ),
-        all_configured_hosts=[
-            HostName("host1"),
-            HostName("host2"),
-            HostName("host3"),
-        ],
+        all_configured_hosts=frozenset(
+            [
+                HostName("host1"),
+                HostName("host2"),
+                HostName("host3"),
+            ]
+        ),
         clusters_of={},
         nodes_of={},
     )