move manpages to plugin families

We want to have all files related to a plugin family in the respective
folder.

This patch provides the required tools, and moves a first couple of
manpages. The selection is choosen such that the tests become simpler,
the rest will follow.

Change-Id: I11c10525dc01c07ad2d489e1354610073928f284

bin/mkp

@@ -28,7 +28,7 @@ _PATH_CONFIG = PathConfig(
     agents_dir=cmk.utils.paths.local_agents_dir,
     alert_handlers_dir=cmk.utils.paths.local_alert_handlers_dir,
     bin_dir=cmk.utils.paths.local_bin_dir,
-    check_manpages_dir=cmk.utils.paths.local_check_manpages_dir,
+    check_manpages_dir=cmk.utils.paths.local_legacy_check_manpages_dir,
     checks_dir=cmk.utils.paths.local_checks_dir,
     doc_dir=cmk.utils.paths.local_doc_dir,
     gui_plugins_dir=cmk.utils.paths.local_gui_plugins_dir,

cmk/base/automations/check_mk.py

@@ -166,6 +166,7 @@
 from cmk.base.sources import make_parser
 
 from cmk.agent_based.v1.value_store import set_value_store_manager
+from cmk.discover_plugins import discover_families, PluginGroup
 
 HistoryFile = str
 HistoryFilePair = tuple[HistoryFile, HistoryFile]
@@ -1681,7 +1682,9 @@ class AutomationGetCheckInformation(Automation):
     needs_checks = True
 
     def execute(self, args: list[str]) -> GetCheckInformationResult:
-        man_page_path_map = man_pages.make_man_page_path_map(man_pages.get_man_page_dirs())
+        man_page_path_map = man_pages.make_man_page_path_map(
+            discover_families(raise_errors=cmk.utils.debug.enabled()), PluginGroup.CHECKMAN.value
+        )
 
         plugin_infos: dict[CheckPluginNameStr, dict[str, Any]] = {}
         for plugin in agent_based_register.iter_all_check_plugins():

cmk/base/modes/check_mk.py

@@ -125,6 +125,7 @@
 from cmk.base.sources import make_parser
 
 from cmk.agent_based.v1.value_store import set_value_store_manager
+from cmk.discover_plugins import discover_families, PluginGroup
 
 from ._localize import do_localize
 
@@ -446,7 +447,9 @@ def mode_list_checks() -> None:
 
     all_check_manuals = {
         n: man_pages.parse_man_page(n, p)
-        for n, p in man_pages.make_man_page_path_map(man_pages.get_man_page_dirs()).items()
+        for n, p in man_pages.make_man_page_path_map(
+            discover_families(raise_errors=cmk.utils.debug.enabled()), PluginGroup.CHECKMAN.value
+        ).items()
     }
 
     all_checks: list[CheckPluginName | str] = [
@@ -1433,7 +1436,9 @@ def mode_reload(args: Sequence[HostName]) -> None:
 def mode_man(options: Mapping[str, str], args: list[str]) -> None:
     import cmk.utils.man_pages as man_pages  # pylint: disable=import-outside-toplevel
 
-    man_page_path_map = man_pages.make_man_page_path_map(man_pages.get_man_page_dirs())
+    man_page_path_map = man_pages.make_man_page_path_map(
+        discover_families(raise_errors=cmk.utils.debug.enabled()), PluginGroup.CHECKMAN.value
+    )
     if not args:
         man_pages.print_man_page_table(man_page_path_map)
         return
@@ -1501,7 +1506,11 @@ def mode_man(options: Mapping[str, str], args: list[str]) -> None:
 def mode_browse_man() -> None:
     import cmk.utils.man_pages as man_pages  # pylint: disable=import-outside-toplevel
 
-    man_pages.print_man_page_browser(man_pages.load_man_page_catalog(man_pages.get_man_page_dirs()))
+    man_pages.print_man_page_browser(
+        man_pages.load_man_page_catalog(
+            discover_families(raise_errors=cmk.utils.debug.enabled()), PluginGroup.CHECKMAN.value
+        )
+    )
 
 
 modes.register(

cmk/discover_plugins.py

@@ -31,10 +31,11 @@
 class PluginGroup(enum.Enum):
     """Definitive list of discoverable plugin groups"""
 
-    GRAPHING = "graphing"
     AGENT_BASED = "agent_based"
-    SERVER_SIDE_CALLS = "server_side_calls"
+    CHECKMAN = "checkman"
+    GRAPHING = "graphing"
     RULESETS = "rulesets"
+    SERVER_SIDE_CALLS = "server_side_calls"
 
 
 class _PluginProtocol(Protocol):

cmk/gui/painter/v0/painters.py

@@ -66,6 +66,8 @@
 )
 from cmk.gui.visual_link import render_link_to_view
 
+from cmk.discover_plugins import discover_families, PluginGroup
+
 from ..v1.helpers import get_perfdata_nth_value, is_stale, paint_stalified
 from .base import Cell, Painter, PainterRegistry
 from .helpers import (
@@ -1542,7 +1544,9 @@ def render(self, row: Row, cell: Cell) -> CellSpec:
         else:
             checktype = command[9:]
 
-        man_page_path_map = man_pages.make_man_page_path_map(man_pages.get_man_page_dirs())
+        man_page_path_map = man_pages.make_man_page_path_map(
+            discover_families(raise_errors=False), PluginGroup.CHECKMAN.value
+        )
         # some checks are run as commandlines (e.g. checks configured via the "Integrate nagios plugins" rule).
         name = checktype.split()[0]
 

cmk/gui/wato/pages/check_catalog.py

@@ -11,7 +11,6 @@
 
 import re
 from collections.abc import Collection, Mapping, Sequence
-from pathlib import Path
 from typing import Any, overload
 
 import cmk.utils.man_pages as man_pages
@@ -46,6 +45,8 @@
 from cmk.gui.watolib.mode import ModeRegistry, WatoMode
 from cmk.gui.watolib.rulespecs import rulespec_registry
 
+from cmk.discover_plugins import discover_families, PluginGroup
+
 from ._tile_menu import TileMenuRenderer
 
 
@@ -66,7 +67,7 @@ def static_permissions() -> Collection[PermissionName]:
         return ["check_plugins"]
 
     def _from_vars(self):
-        self._manpages = _get_check_catalog(man_pages.get_man_page_dirs(), only_path=())
+        self._manpages = _get_check_catalog(discover_families(raise_errors=False), only_path=())
         self._titles = man_pages.CATALOG_TITLES
 
     def title(self) -> str:
@@ -120,7 +121,7 @@ def parent_mode(cls) -> type[WatoMode] | None:
 
     def _from_vars(self):
         self._search = get_search_expression()
-        self._manpages = _get_check_catalog(man_pages.get_man_page_dirs(), only_path=())
+        self._manpages = _get_check_catalog(discover_families(raise_errors=False), only_path=())
         self._titles = man_pages.CATALOG_TITLES
 
     def title(self) -> str:
@@ -235,7 +236,7 @@ def _from_vars(self):
         for comp in self._path:
             ID().validate_value(comp, None)  # Beware against code injection!
 
-        self._manpages = _get_check_catalog(man_pages.get_man_page_dirs(), self._path)
+        self._manpages = _get_check_catalog(discover_families(raise_errors=False), self._path)
         self._titles = man_pages.CATALOG_TITLES
 
         self._has_second_level = None
@@ -403,7 +404,7 @@ def _man_page_catalog_topics():
 
 
 def _get_check_catalog(
-    man_page_dirs: Sequence[Path],
+    plugin_families: Mapping[str, Sequence[str]],
     only_path: tuple[str, ...],
 ) -> Mapping[str, Any]:
     # Note: this is impossible to type, since the type is recursive.
@@ -415,7 +416,9 @@ def path_prefix_matches(p: tuple[str, ...]) -> bool:
 
     tree: dict[str, Any] = {}
 
-    for path, entries in man_pages.load_man_page_catalog(man_page_dirs).items():
+    for path, entries in man_pages.load_man_page_catalog(
+        plugin_families, PluginGroup.CHECKMAN.value
+    ).items():
         if not path_prefix_matches(path):
             continue
         subtree = tree
@@ -469,7 +472,9 @@ def _from_vars(self) -> None:
         ):
             raise MKUserError("check_type", _("Invalid check type"))
 
-        man_page_paths = man_pages.make_man_page_path_map(man_pages.get_man_page_dirs())
+        man_page_paths = man_pages.make_man_page_path_map(
+            discover_families(raise_errors=False), PluginGroup.CHECKMAN.value
+        )
 
         try:
             man_page_path = man_page_paths[self._check_plugin_name]

cmk/gui/watolib/activate_changes.py

@@ -2584,7 +2584,7 @@ def _execute_post_config_sync_actions(site_id: SiteId) -> None:
                     agents_dir=paths.local_agents_dir,
                     alert_handlers_dir=paths.local_alert_handlers_dir,
                     bin_dir=paths.local_bin_dir,
-                    check_manpages_dir=paths.local_check_manpages_dir,
+                    check_manpages_dir=paths.local_legacy_check_manpages_dir,
                     checks_dir=paths.local_checks_dir,
                     doc_dir=paths.local_doc_dir,
                     gui_plugins_dir=paths.local_gui_plugins_dir,

cmk/gui/werks.py

@@ -15,7 +15,7 @@
 from typing_extensions import TypedDict
 
 import cmk.utils.werks.werk as utils_werks_werk
-from cmk.utils.man_pages import get_man_page_dirs, make_man_page_path_map
+from cmk.utils.man_pages import make_man_page_path_map
 from cmk.utils.version import __version__, Edition, Version
 from cmk.utils.werks.acknowledgement import is_acknowledged
 from cmk.utils.werks.acknowledgement import load_acknowledgements as werks_load_acknowledgements
@@ -70,6 +70,7 @@
     ValueSpec,
 )
 
+from cmk.discover_plugins import discover_families, PluginGroup
 from cmk.werks.models import Compatibility, Werk
 
 TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -869,4 +870,4 @@ def insert_manpage_links(text: str) -> HTML:
 
 @request_memoize()
 def _get_known_checks() -> Container[str]:
-    return make_man_page_path_map(get_man_page_dirs())
+    return make_man_page_path_map(discover_families(raise_errors=False), PluginGroup.CHECKMAN.value)

cmk/update_config/plugins/pre_actions/utils.py

@@ -34,7 +34,7 @@
     agents_dir=paths.local_agents_dir,
     alert_handlers_dir=paths.local_alert_handlers_dir,
     bin_dir=paths.local_bin_dir,
-    check_manpages_dir=paths.local_check_manpages_dir,
+    check_manpages_dir=paths.local_legacy_check_manpages_dir,
     checks_dir=paths.local_checks_dir,
     doc_dir=paths.local_doc_dir,
     gui_plugins_dir=paths.local_gui_plugins_dir,

cmk/utils/man_pages.py

@@ -15,7 +15,7 @@
 import subprocess
 import sys
 from collections import defaultdict
-from collections.abc import Iterable, Mapping, Reversible, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 from contextlib import suppress
 from dataclasses import dataclass
 from io import StringIO
@@ -28,6 +28,12 @@
 from cmk.utils.exceptions import MKGeneralException
 from cmk.utils.i18n import _
 
+# remove with 2.4 / after 2.3 is released
+_LEGACY_MAN_PAGE_PATHS = (
+    str(cmk.utils.paths.local_legacy_check_manpages_dir),
+    cmk.utils.paths.legacy_check_manpages_dir,
+)
+
 
 @dataclass
 class ManPage:
@@ -318,22 +324,25 @@ def fallback(cls, path: Path, name: str, msg: str, content: str) -> "ManPage":
 }
 
 
-def get_man_page_dirs() -> Sequence[Path]:
-    # first match wins
-    return [
-        cmk.utils.paths.local_check_manpages_dir,
-        Path(cmk.utils.paths.check_manpages_dir),
-    ]
-
-
 def _is_valid_basename(name: str) -> bool:
     return not name.startswith(".") and not name.endswith("~")
 
 
-def make_man_page_path_map(man_page_dirs: Reversible[Path]) -> Mapping[str, Path]:
+def make_man_page_path_map(
+    plugin_families: Mapping[str, Sequence[str]],
+    group_subdir: str,
+) -> Mapping[str, Path]:
+    families_man_paths = [
+        *(
+            os.path.join(p, group_subdir)
+            for _family, paths in plugin_families.items()
+            for p in paths
+        ),
+        *_LEGACY_MAN_PAGE_PATHS,
+    ]
     return {
         name: Path(dir, name)
-        for source in reversed(man_page_dirs)
+        for source in reversed(families_man_paths)
         for dir, _subdirs, files in os.walk(source)
         for name in files
         if _is_valid_basename(name)
@@ -359,8 +368,10 @@ def get_title_from_man_page(path: Path) -> str:
     raise MKGeneralException(_("Invalid man page: Failed to get the title"))
 
 
-def load_man_page_catalog(man_page_dirs: Reversible[Path]) -> ManPageCatalog:
-    man_page_path_map = make_man_page_path_map(man_page_dirs)
+def load_man_page_catalog(
+    plugin_families: Mapping[str, Sequence[str]], group_subdir: str
+) -> ManPageCatalog:
+    man_page_path_map = make_man_page_path_map(plugin_families, group_subdir)
     catalog: dict[ManPageCatalogPath, list[ManPage]] = defaultdict(list)
     for name, path in man_page_path_map.items():
         parsed = parse_man_page(name, path)
@@ -838,15 +849,16 @@ def _apply_markup(line: str) -> str:
 
 
 def man_pages_for_website_export(
-    man_page_paths: Reversible[Path],
+    plugin_families: Mapping[str, Sequence[str]],
+    group_subdir: str,
 ) -> dict[str, _SerializableManPageDerivative]:
     """This is called from `scripts/create_man_pages.py` of the websites-essentials repo!
 
     The result should be json serializable.
     """
     return {
         name: _extend_categorization_info(parse_man_page(name, path))
-        for name, path in make_man_page_path_map(man_page_paths).items()
+        for name, path in make_man_page_path_map(plugin_families, group_subdir).items()
     }
 
 

cmk/utils/paths.py

@@ -99,7 +99,7 @@ def _local_path(global_path: str | Path) -> Path:
 checks_dir = _omd_path_str("share/check_mk/checks")
 notifications_dir = _omd_path("share/check_mk/notifications")
 inventory_dir = _omd_path_str("share/check_mk/inventory")
-check_manpages_dir = _omd_path_str("share/check_mk/checkman")
+legacy_check_manpages_dir = _omd_path_str("share/check_mk/checkman")
 agents_dir = _omd_path_str("share/check_mk/agents")
 web_dir = _omd_path_str("share/check_mk/web")
 pnp_templates_dir = _omd_path("share/check_mk/pnp-templates")
@@ -125,7 +125,7 @@ def _local_path(global_path: str | Path) -> Path:
 local_agent_based_plugins_dir = _local_path(agent_based_plugins_dir)
 local_notifications_dir = _local_path(notifications_dir)
 local_inventory_dir = _local_path(inventory_dir)
-local_check_manpages_dir = _local_path(check_manpages_dir)
+local_legacy_check_manpages_dir = _local_path(legacy_check_manpages_dir)
 local_agents_dir = _local_path(agents_dir)
 local_web_dir = _local_path(web_dir)
 local_pnp_templates_dir = _local_path(pnp_templates_dir)

omd/packages/check_mk/check_mk.make

@@ -103,9 +103,6 @@ $(CHECK_MK_INTERMEDIATE_INSTALL): $(SOURCE_BUILT_AGENTS) $(CHECK_MK_BUILD) $(PAC
 	    --transform "s/^plugin-api\/build/plugin-api/" \
 	    plugin-api/build/html | tar -x -C $(CHECK_MK_INSTALL_DIR)/share/doc/check_mk/
 
-	$(MKDIR) $(CHECK_MK_INSTALL_DIR)/share/check_mk/checkman
-	install -m 644 $(REPO_PATH)/checkman/* $(CHECK_MK_INSTALL_DIR)/share/check_mk/checkman
-
 	$(MKDIR) $(CHECK_MK_INSTALL_DIR)/share/check_mk/agents
 	tar -c -C $(REPO_PATH)/agents \
 	    $(CHECK_MK_TAROPTS) \

tests/code_quality/file_content/test_permissions.py

@@ -39,7 +39,8 @@ def is_not_executable(path: Path) -> bool:
         ["*.checksum", "*.pyc"],
     ),
     ("checks/*", is_not_executable, [], []),
-    ("checkman/*", is_not_executable, [], []),
+    ("cmk/plugins/*/manpages/*", is_not_executable, [], []),
+    ("cmk/plugins/*/manpages/*/*", is_executable, [], []),  # THIS SHOULD FAIL
     ("pnp-templates/*", is_not_executable, [], []),
     ("notifications/*", is_executable, ["README", "debug"], []),
     ("bin/*", is_executable, ["Makefile", "mkevent.cc", "mkeventd_open514.cc"], []),

tests/testlib/__init__.py

@@ -126,7 +126,7 @@ def fake_version_and_paths() -> None:
     monkeypatch.setattr("cmk.utils.paths.checks_dir", "%s/checks" % cmk_path())
     monkeypatch.setattr("cmk.utils.paths.notifications_dir", Path(cmk_path()) / "notifications")
     monkeypatch.setattr("cmk.utils.paths.inventory_dir", "%s/inventory" % cmk_path())
-    monkeypatch.setattr("cmk.utils.paths.check_manpages_dir", "%s/checkman" % cmk_path())
+    monkeypatch.setattr("cmk.utils.paths.legacy_check_manpages_dir", "%s/checkman" % cmk_path())
     monkeypatch.setattr("cmk.utils.paths.web_dir", "%s/web" % cmk_path())
 
 

tests/testlib/site.py

@@ -700,7 +700,6 @@ def _update_with_f12_files(self) -> None:
             cmk_path() + "/agents",
             cmk_path() + "/cmk/base",
             cmk_path() + "/cmk",
-            cmk_path() + "/checkman",
             cmk_path() + "/web",
             cmk_path() + "/inventory",
             cmk_path() + "/notifications",

tests/unit/cmk/utils/licensing/test_usage.py

@@ -34,6 +34,8 @@
     try_update_license_usage,
 )
 
+from cmk.discover_plugins import discover_families, PluginGroup
+
 
 def test_try_update_license_usage() -> None:
     instance_id = UUID("937495cb-78f7-40d4-9b5f-f2c5a81e66b8")
@@ -795,7 +797,9 @@ def is_cloud_manpage(catalog_path: man_pages.ManPageCatalogPath) -> bool:
             catalog_path[0] == "cloud" and catalog_path[1] not in not_cloud_for_licensing_purposes
         )
 
-    catalog = man_pages.load_man_page_catalog(man_pages.get_man_page_dirs())
+    catalog = man_pages.load_man_page_catalog(
+        discover_families(raise_errors=True), PluginGroup.CHECKMAN.value
+    )
 
     cloud_man_pages = [
         manpage