Add locking of requirements_lock.txt

* at the moment we may have a requirements_lock beside a Pipfile.lock
* so lock them as well via bazel

Change-Id: Idf3dfb5ae5ede012316d443990c0f87f938a42a6

BUILD

@@ -47,3 +47,8 @@ refresh_compile_commands(
         "//packages/unixcat:all": "",
     },
 )
+
+alias(
+    name = "requirements.update",
+    actual = "//cmk:requirements.update",
+)

cmk/BUILD

@@ -31,4 +31,5 @@ compile_pip_requirements(
     name = "requirements",
     requirements_in = "requirements_in.txt",
     requirements_txt = "@//:requirements_lock.txt",
+    visibility = ["//visibility:public"],
 )

scripts/update-pipfiles.py

@@ -12,6 +12,7 @@
 import subprocess
 import threading
 from collections.abc import Sequence
+from dataclasses import dataclass
 from itertools import chain
 from pathlib import Path
 from typing import NamedTuple
@@ -28,16 +29,33 @@ class UpdateInfo(NamedTuple):
     to_version: str
 
 
-def _find_pipfile_locks() -> Sequence[Path]:
-    """return a list of paths where we found Pipfile.locks"""
+@dataclass
+class LockPaths:
+    pipfile_lock: Path
+    requirement_lock: Path | None = None
+
+    def __init__(self, p: Path):
+        self.pipfile_lock = p
+        requirements_lock = Path(p.parents[0] / "requirements_lock.txt")
+        if requirements_lock.exists():
+            self.requirement_lock = requirements_lock
+
+
+class CommitInfo(NamedTuple):
+    file_to_be_committed: Path
+    update_info: Sequence[UpdateInfo]
+
+
+def _find_locks() -> Sequence[LockPaths]:
+    """return a list of paths where we found Pipfile.locks and - if residing next to it - a requirements_lock.txt"""
     return sorted(
         (
-            Path(p)
+            LockPaths(Path(p))
             for p in subprocess.check_output(
                 ["find", ".", "-name", "Pipfile.lock", "-type", "f"], text=True
             ).splitlines()
         ),
-        key=lambda p: len(p.parents),
+        key=lambda p: len(p.pipfile_lock.parents),
         reverse=True,
     )
 
@@ -56,6 +74,16 @@ def _parse_pipenv_lock(piplock_path: Path) -> dict[str, str]:
     }
 
 
+def _parse_requirements_lock(requirements_lock: Path) -> dict[str, str]:
+    """return a dictionary of package names to versions"""
+    with requirements_lock.open() as requirements_lock_file:
+        return {
+            line.split("==")[0]: line.split("==")[1].split()[0].strip()
+            for line in requirements_lock_file
+            if not line.startswith("#") and "==" in line
+        }
+
+
 def _diff_versions(before: dict[str, str], after: dict[str, str]) -> Sequence[UpdateInfo]:
     """diff to parsed pipenv lock files"""
     return [
@@ -83,34 +111,61 @@ def _update_piplock(piplock_path: Path) -> Sequence[UpdateInfo]:
     return changed_versions
 
 
-def _commit_piplock(piplock_path: Path, version_diff: Sequence[UpdateInfo]) -> None:
+def _update_requirementslock(requirements_lock: Path) -> Sequence[UpdateInfo]:
+    logging.info("Updating %s", requirements_lock)
+    before_versions = _parse_requirements_lock(requirements_lock)
     subprocess.run(
-        ["git", "add", "Pipfile.lock"],
-        cwd=piplock_path.parent,
+        ["bazel", "run", ":requirements.update"],
+        cwd=requirements_lock.parent,
         check=True,
+        env=environment,
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
     )
+    after_versions = _parse_requirements_lock(requirements_lock)
+    changed_versions = _diff_versions(before_versions, after_versions)
+    for info in changed_versions:
+        logging.info("%s: %s -> %s", info.name, info.from_version, info.to_version)
+    return changed_versions
 
-    message = f"pipenv lock{"" if (p := piplock_path.parent) == Path() else f" in {p}"}\n\n"
-    message += "\n".join(
-        f"{info.name}: {info.from_version} -> {info.to_version}" for info in version_diff
-    )
+
+def _commit_lock(commit_infos: Sequence[CommitInfo]) -> None:
+    message = f"lock python dependencies {"" if (p := commit_infos[0].file_to_be_committed.parent) == Path() else f" in {p}"}\n\n"
+    for ci in commit_infos:
+        subprocess.run(
+            ["git", "add", ci.file_to_be_committed],
+            cwd=ci.file_to_be_committed.parent,
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+        message += f"\n{ci.file_to_be_committed.name} update:\n"
+        message += "\n".join(
+            f"* {info.name}: {info.from_version} -> {info.to_version}" for info in ci.update_info
+        )
     message += "\n"
 
     subprocess.run(
         ["git", "commit", "-m", message],
-        cwd=piplock_path.parent,
+        cwd=commit_infos[0].file_to_be_committed.parent,
         check=True,
     )
 
 
-def _handle_piplock(task_queue: queue.Queue[Path], lock: threading.Lock) -> None:
+def _handle_lock_files(task_queue: queue.Queue[LockPaths], lock: threading.Lock) -> None:
     while True:
-        piplock_path = task_queue.get()
-        if version_diff := _update_piplock(piplock_path):
+        lock_files = task_queue.get()
+        commit_infos = []
+        if lock_files.requirement_lock is not None and (
+            version_diff_req := _update_requirementslock(lock_files.requirement_lock)
+        ):
+            commit_infos.append(CommitInfo(lock_files.requirement_lock, version_diff_req))
+        if version_diff_pipfile := _update_piplock(lock_files.pipfile_lock):
+            commit_infos.append(CommitInfo(lock_files.pipfile_lock, version_diff_pipfile))
+        if commit_infos:
             with lock:  # only one thread should commit at a time
-                _commit_piplock(piplock_path, version_diff)
+                _commit_lock(commit_infos)
 
         task_queue.task_done()
 
@@ -120,12 +175,12 @@ def _handle_piplock(task_queue: queue.Queue[Path], lock: threading.Lock) -> None
     logging.basicConfig(level=logging.INFO)
 
     commit_lock = threading.Lock()
-    tasks = queue.Queue[Path]()
+    tasks = queue.Queue[LockPaths]()
 
     for _ in range(N_THREADS):
-        threading.Thread(target=_handle_piplock, args=(tasks, commit_lock), daemon=True).start()
+        threading.Thread(target=_handle_lock_files, args=(tasks, commit_lock), daemon=True).start()
 
-    for path in _find_pipfile_locks():
-        tasks.put(path)
+    for lock_paths in _find_locks():
+        tasks.put(lock_paths)
 
     tasks.join()