diff --git a/.werks/14237.md b/.werks/14237.md new file mode 100644 index 00000000000..819b5f8950b --- /dev/null +++ b/.werks/14237.md @@ -0,0 +1,15 @@ +[//]: # (werk v2) +# Fixed performance problems for various command line operations + +key | value +---------- | --- +date | 2024-09-27T10:40:03+00:00 +version | 2.3.0p18 +class | fix +edition | cre +component | checks +level | 1 +compatible | yes + +With larger setups (>10000 hosts), loading the host configuration took considerably longer ( **O(n^2)** ) than normal. +The ‘Parameters for this service’ page, for example, requires the complete host configuration. diff --git a/cmk/base/config.py b/cmk/base/config.py index 5a73f097bf6..47435027594 100644 --- a/cmk/base/config.py +++ b/cmk/base/config.py @@ -40,18 +40,13 @@ import cmk.utils.check_utils import cmk.utils.cleanup import cmk.utils.config_path -import cmk.utils.config_warnings as config_warnings import cmk.utils.debug -import cmk.utils.password_store as password_store import cmk.utils.paths -import cmk.utils.piggyback as piggyback -import cmk.utils.rulesets.ruleset_matcher as ruleset_matcher -import cmk.utils.rulesets.tuple_rulesets as tuple_rulesets -import cmk.utils.store as store import cmk.utils.store.host_storage import cmk.utils.tags import cmk.utils.translations import cmk.utils.version as cmk_version +from cmk.utils import config_warnings, password_store, piggyback, store from cmk.utils.agent_registration import ( connection_mode_from_host_config, HostAgentConnectionMode, @@ -80,7 +75,7 @@ from cmk.utils.log import console from cmk.utils.macros import replace_macros_in_str from cmk.utils.regex import regex -from cmk.utils.rulesets import RuleSetName +from cmk.utils.rulesets import ruleset_matcher, RuleSetName, tuple_rulesets from cmk.utils.rulesets.ruleset_matcher import ( LabelManager, LabelSources, @@ -152,8 +147,7 @@ ) import cmk.base.api.agent_based.register as agent_based_register -import cmk.base.default_config as default_config -import cmk.base.ip_lookup as ip_lookup +from cmk.base import default_config, ip_lookup from cmk.base.api.agent_based.cluster_mode import ClusterMode from cmk.base.api.agent_based.plugin_classes import SNMPSectionPlugin from cmk.base.api.agent_based.register.check_plugins_legacy import ( @@ -2022,13 +2016,11 @@ def initialize(self) -> ConfigCache: ), clusters_of=self._clusters_of_cache, nodes_of=self._nodes_of_cache, - all_configured_hosts=list( - set( - itertools.chain( - self.hosts_config.hosts, - self.hosts_config.clusters, - self.hosts_config.shadow_hosts, - ) + all_configured_hosts=frozenset( + itertools.chain( + self.hosts_config.hosts, + self.hosts_config.clusters, + self.hosts_config.shadow_hosts, ) ), ) diff --git a/cmk/utils/rulesets/ruleset_matcher.py b/cmk/utils/rulesets/ruleset_matcher.py index 1ab062b38aa..6fb9987854d 100644 --- a/cmk/utils/rulesets/ruleset_matcher.py +++ b/cmk/utils/rulesets/ruleset_matcher.py @@ -8,7 +8,7 @@ import dataclasses from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from re import Pattern -from typing import Any, cast, Generic, NamedTuple, NotRequired, TypeAlias, TypeVar +from typing import Any, cast, FrozenSet, Generic, NamedTuple, NotRequired, Set, TypeAlias, TypeVar from typing_extensions import TypedDict @@ -150,7 +150,7 @@ def __init__( host_tags: TagsOfHosts, host_paths: Mapping[HostName, str], label_manager: LabelManager, - all_configured_hosts: Sequence[HostName], + all_configured_hosts: FrozenSet[HostName], clusters_of: Mapping[HostName, Sequence[HostName]], nodes_of: Mapping[HostName, Sequence[HostName]], ) -> None: @@ -400,7 +400,7 @@ def __init__( host_tags: TagsOfHosts, host_paths: Mapping[HostName, str], label_manager: LabelManager, - all_configured_hosts: Sequence[HostName], + all_configured_hosts: FrozenSet[HostName], clusters_of: Mapping[HostName, Sequence[HostName]], nodes_of: Mapping[HostName, Sequence[HostName]], ) -> None: @@ -450,11 +450,11 @@ def clear_caches(self) -> None: self.__host_ruleset_cache.clear() self._all_matching_hosts_match_cache.clear() - def all_processed_hosts(self) -> Sequence[HostName]: + def all_processed_hosts(self) -> FrozenSet[HostName]: """Returns a set of all processed hosts""" return self._all_processed_hosts - def set_all_processed_hosts(self, all_processed_hosts: Iterable[HostName]) -> None: + def set_all_processed_hosts(self, all_processed_hosts: Set[HostName]) -> None: involved_clusters: set[HostName] = set() involved_nodes: set[HostName] = set() for hostname in self._all_processed_hosts: @@ -465,11 +465,11 @@ def set_all_processed_hosts(self, all_processed_hosts: Iterable[HostName]) -> No for hostname in involved_clusters: involved_nodes.update(self._nodes_of.get(hostname, [])) - nodes_and_clusters = involved_clusters | involved_nodes | set(all_processed_hosts) + nodes_and_clusters = involved_clusters | involved_nodes | all_processed_hosts # Only add references to configured hosts nodes_and_clusters.intersection_update(self._all_configured_hosts) - self._all_processed_hosts = list(nodes_and_clusters) + self._all_processed_hosts = frozenset(nodes_and_clusters) # The folder host lookup includes a list of all -processed- hosts within a given # folder. Any update with set_all_processed hosts invalidates this cache, because diff --git a/tests/unit/cmk/utils/rulesets/test_ruleset_matcher.py b/tests/unit/cmk/utils/rulesets/test_ruleset_matcher.py index 151e3556e95..e0cd4debb37 100644 --- a/tests/unit/cmk/utils/rulesets/test_ruleset_matcher.py +++ b/tests/unit/cmk/utils/rulesets/test_ruleset_matcher.py @@ -146,7 +146,7 @@ def test_ruleset_matcher_get_host_values_labels( service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[HostName("host1"), HostName("host2")], + all_configured_hosts=frozenset([HostName("host1"), HostName("host2")]), clusters_of={}, nodes_of={}, ) @@ -183,7 +183,7 @@ def test_labels_of_service(monkeypatch: MonkeyPatch) -> None: ], discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[test_host, xyz_host], + all_configured_hosts=frozenset([test_host, xyz_host]), clusters_of={}, nodes_of={}, ) @@ -215,7 +215,7 @@ def test_labels_of_service_discovered_labels() -> None: lambda host_name, *args, **kw: {"äzzzz": "eeeeez"} if host_name == test_host else {} ), ), - all_configured_hosts=[test_host], + all_configured_hosts=frozenset([test_host]), clusters_of={}, nodes_of={}, ) @@ -246,12 +246,14 @@ def test_basic_get_host_values() -> None: service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[ - HostName("abc"), - HostName("xyz"), - HostName("host1"), - HostName("host2"), - ], + all_configured_hosts=frozenset( + [ + HostName("abc"), + HostName("xyz"), + HostName("host1"), + HostName("host2"), + ] + ), clusters_of={}, nodes_of={}, ) @@ -285,13 +287,15 @@ def test_basic_get_host_values_subfolders() -> None: service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[ - HostName("abc"), - HostName("xyz"), - HostName("lvl1"), - HostName("lvl2"), - HostName("lvl1a"), - ], + all_configured_hosts=frozenset( + [ + HostName("abc"), + HostName("xyz"), + HostName("lvl1"), + HostName("lvl2"), + HostName("lvl1a"), + ] + ), clusters_of={}, nodes_of={}, ) @@ -361,12 +365,14 @@ def test_basic_host_ruleset_get_merged_dict_values() -> None: service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[ - HostName("abc"), - HostName("xyz"), - HostName("host1"), - HostName("host2"), - ], + all_configured_hosts=frozenset( + [ + HostName("abc"), + HostName("xyz"), + HostName("host1"), + HostName("host2"), + ] + ), clusters_of={}, nodes_of={}, ) @@ -436,12 +442,14 @@ def test_basic_host_ruleset_get_host_bool_value() -> None: service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[ - HostName("abc"), - HostName("xyz"), - HostName("host1"), - HostName("host2"), - ], + all_configured_hosts=frozenset( + [ + HostName("abc"), + HostName("xyz"), + HostName("host1"), + HostName("host2"), + ] + ), clusters_of={}, nodes_of={}, ) @@ -559,11 +567,13 @@ def test_ruleset_matcher_get_host_values_tags( service_label_rules=(), discovered_labels_of_service=lambda *args, **kw: {}, ), - all_configured_hosts=[ - HostName("host1"), - HostName("host2"), - HostName("host3"), - ], + all_configured_hosts=frozenset( + [ + HostName("host1"), + HostName("host2"), + HostName("host3"), + ] + ), clusters_of={}, nodes_of={}, )