add common plugin loading

Change-Id: Iea7f9e5c023bebf8a7631891fd2b393273cb2a57

cmk/base/plugins/config_generation/register.py

@@ -6,27 +6,21 @@
 from collections.abc import Mapping, Sequence
 
 import cmk.utils.debug
-from cmk.utils.plugin_loader import load_plugins
 
 from cmk.config_generation.v1 import ActiveCheckConfig
+from cmk.discover_plugins import discover_plugins
 
 
 def load_active_checks() -> tuple[Sequence[str], Mapping[str, ActiveCheckConfig]]:
-    errors = []  # TODO: see if we really need to return the errors.
-    # Maybe we can just either ignore or raise them.
-
-    registered_active_checks = {}
-    for plugin, exception_or_module in load_plugins(
-        "cmk.base.plugins.config_generation.active_checks"
-    ):
-        match exception_or_module:
-            case BaseException() as exc:
-                if cmk.utils.debug.enabled():
-                    raise exc
-                errors.append(f"Error in active check plugin {plugin}: {exc}\n")
-            case module:
-                for name, value in vars(module).items():
-                    if name.startswith("active_check") and isinstance(value, ActiveCheckConfig):
-                        registered_active_checks[value.name] = value
-
-    return errors, registered_active_checks
+    loaded = discover_plugins(
+        "config_generation",
+        "active_check_",
+        ActiveCheckConfig,
+        raise_errors=cmk.utils.debug.enabled(),
+    )
+    # TODO:
+    #  * see if we really need to return the errors. Maybe we can just either ignore or raise them.
+    #  * deal with duplicate names.
+    return [str(e) for e in loaded.errors], {
+        plugin.name: plugin for plugin in loaded.plugins.values()
+    }

cmk/discover_plugins.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+"""Loading API based plugins from cmk.plugins
+
+This implements common logic for loading API based plugins
+(yes, we have others) from cmk.plugins.
+
+We have more "plugin" loading logic else where, but there
+are subtle differences with respect to the treatment of
+namespace packages and error handling.
+
+Changes in this file might result in different behaviour
+of plugins developed against a versionized API.
+
+Please keep this in mind when trying to consolidate.
+"""
+import importlib
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from types import ModuleType
+from typing import Generic, TypeVar
+
+PLUGIN_BASE = "cmk.plugins"
+
+
+_PluginType = TypeVar("_PluginType")
+
+
+@dataclass(frozen=True)
+class PluginLocation:
+    module: str
+    name: str | None = None
+
+    def __str__(self) -> str:
+        return f"{self.module}:{self.name}"
+
+
+@dataclass(frozen=True)
+class DiscoveredPlugins(Generic[_PluginType]):
+    errors: Sequence[Exception]
+    plugins: Mapping[PluginLocation, _PluginType]
+
+
+def discover_plugins(
+    plugin_group: str,
+    name_prefix: str,
+    plugin_type: type[_PluginType],
+    *,
+    raise_errors: bool,
+) -> DiscoveredPlugins[_PluginType]:
+    """Load all specified packages"""
+
+    try:
+        plugin_base = importlib.import_module(PLUGIN_BASE)
+    except Exception as exc:
+        if raise_errors:
+            raise
+        return DiscoveredPlugins((exc,), {})
+
+    errors = []
+    plugins: dict[PluginLocation, _PluginType] = {}
+    for pkg_name in _find_namespaces(plugin_base, plugin_group):
+        try:
+            module = importlib.import_module(pkg_name)
+        except ModuleNotFoundError:
+            pass  # don't choke upon empty folders.
+        except Exception as exc:
+            if raise_errors:
+                raise
+            errors.append(exc)
+            continue
+
+        module_errors, module_plugins = _collect_module_plugins(
+            pkg_name, vars(module), name_prefix, plugin_type, raise_errors
+        )
+        errors.extend(module_errors)
+        plugins.update(module_plugins)
+
+    return DiscoveredPlugins(errors, plugins)
+
+
+def _find_namespaces(plugin_base: ModuleType, plugin_group: str) -> set[str]:
+    return {
+        f"{plugin_base.__name__}.{family}.{plugin_group}.{fname.removesuffix('.py')}"
+        for path in plugin_base.__path__
+        for family in _ls_defensive(path)
+        for fname in _ls_defensive(f"{path}/{family}/{plugin_group}")
+        if fname not in {"__pycache__", "__init__.py"}
+    }
+
+
+def _ls_defensive(path: str) -> Sequence[str]:
+    try:
+        return list(os.listdir(path))
+    except FileNotFoundError:
+        return []
+
+
+def _collect_module_plugins(
+    module_name: str,
+    objects: Mapping[str, object],
+    name_prefix: str,
+    plugin_type: type[_PluginType],
+    raise_errors: bool,
+) -> tuple[Sequence[Exception], Mapping[PluginLocation, _PluginType]]:
+    """Dispatch valid and invalid well-known objects
+
+    >>> errors, plugins = _collect_module_plugins("my_module", {"my_plugin": 1, "my_b": "two", "some_c": "ignored"}, "my_", int, False)
+    >>> errors[0]
+    TypeError("my_module:my_b: 'two'")
+    >>> plugins
+    {PluginLocation(module='my_module', name='my_plugin'): 1}
+    """
+    errors = []
+    plugins = {}
+
+    for name, value in objects.items():
+        if not name.startswith(name_prefix):
+            continue
+
+        location = PluginLocation(module_name, name)
+        if isinstance(value, plugin_type):
+            plugins[location] = value
+            continue
+
+        if raise_errors:
+            raise TypeError(f"{location}: {value!r}")
+
+        errors.append(TypeError(f"{location}: {value!r}"))
+
+    return errors, plugins

tests/testlib/pylint_checker_cmk_module_layers.py

@@ -62,7 +62,7 @@ def _in_component(
 
 
 def _is_allowed_import(imported: ModuleName) -> bool:
-    """cmk, cmk.utils, cmk.fields, and cmk.automations are allowed to be imported from all over the place"""
+    """these are allowed to be imported from all over the place"""
     return any(
         (
             imported == "cmk",
@@ -71,6 +71,7 @@ def _is_allowed_import(imported: ModuleName) -> bool:
             _in_component(imported, Component("cmk.automations")),
             _in_component(imported, Component("cmk.bi")),
             _in_component(imported, Component("cmk.config_generation")),
+            _in_component(imported, Component("cmk.discover_plugins")),
         )
     )
 

tests/unit/cmk/base/plugins/commands/active_checks/test_bi_aggr.py

@@ -7,9 +7,8 @@
 
 import pytest
 
-from cmk.base.plugins.config_generation.active_checks.bi_aggr import active_check_bi_aggr
-
 from cmk.config_generation.v1 import ActiveCheckCommand, HostConfig, IPAddressFamily
+from cmk.plugins.collection.config_generation.bi_aggr import active_check_bi_aggr
 
 HOST_CONFIG = HostConfig(
     name="hostname",

tests/unit/cmk/base/plugins/commands/active_checks/test_icmp.py

@@ -7,9 +7,8 @@
 
 import pytest
 
-from cmk.base.plugins.config_generation.active_checks.icmp import active_check_icmp
-
 from cmk.config_generation.v1 import ActiveCheckCommand, HostConfig, IPAddressFamily
+from cmk.plugins.collection.config_generation.icmp import active_check_icmp
 
 HOST_CONFIG = HostConfig(
     name="hostname",