From 81097f0ba5a551d4efeb5307bbe610a638e2b69d Mon Sep 17 00:00:00 2001 From: Timotheus Bachinger Date: Mon, 30 Sep 2024 14:39:51 +0200 Subject: [PATCH] Add locking of requirements_lock.txt * at the moment we may have a requirements_lock beside a Pipfile.lock * so lock them as well via bazel Change-Id: Idf3dfb5ae5ede012316d443990c0f87f938a42a6 --- BUILD | 5 ++ cmk/BUILD | 1 + scripts/update-pipfiles.py | 95 ++++++++++++++++++++++++++++++-------- 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/BUILD b/BUILD index 44125152800..780942b1521 100644 --- a/BUILD +++ b/BUILD @@ -47,3 +47,8 @@ refresh_compile_commands( "//packages/unixcat:all": "", }, ) + +alias( + name = "requirements.update", + actual = "//cmk:requirements.update", +) diff --git a/cmk/BUILD b/cmk/BUILD index 849dc942a0f..69154ca1c17 100644 --- a/cmk/BUILD +++ b/cmk/BUILD @@ -31,4 +31,5 @@ compile_pip_requirements( name = "requirements", requirements_in = "requirements_in.txt", requirements_txt = "@//:requirements_lock.txt", + visibility = ["//visibility:public"], ) diff --git a/scripts/update-pipfiles.py b/scripts/update-pipfiles.py index 41d099d7c50..8330f574d30 100755 --- a/scripts/update-pipfiles.py +++ b/scripts/update-pipfiles.py @@ -12,6 +12,7 @@ import subprocess import threading from collections.abc import Sequence +from dataclasses import dataclass from itertools import chain from pathlib import Path from typing import NamedTuple @@ -28,16 +29,33 @@ class UpdateInfo(NamedTuple): to_version: str -def _find_pipfile_locks() -> Sequence[Path]: - """return a list of paths where we found Pipfile.locks""" +@dataclass +class LockPaths: + pipfile_lock: Path + requirement_lock: Path | None = None + + def __init__(self, p: Path): + self.pipfile_lock = p + requirements_lock = Path(p.parents[0] / "requirements_lock.txt") + if requirements_lock.exists(): + self.requirement_lock = requirements_lock + + +class CommitInfo(NamedTuple): + file_to_be_committed: Path + update_info: Sequence[UpdateInfo] + + +def _find_locks() -> Sequence[LockPaths]: + """return a list of paths where we found Pipfile.locks and - if residing next to it - a requirements_lock.txt""" return sorted( ( - Path(p) + LockPaths(Path(p)) for p in subprocess.check_output( ["find", ".", "-name", "Pipfile.lock", "-type", "f"], text=True ).splitlines() ), - key=lambda p: len(p.parents), + key=lambda p: len(p.pipfile_lock.parents), reverse=True, ) @@ -56,6 +74,16 @@ def _parse_pipenv_lock(piplock_path: Path) -> dict[str, str]: } +def _parse_requirements_lock(requirements_lock: Path) -> dict[str, str]: + """return a dictionary of package names to versions""" + with requirements_lock.open() as requirements_lock_file: + return { + line.split("==")[0]: line.split("==")[1].split()[0].strip() + for line in requirements_lock_file + if not line.startswith("#") and "==" in line + } + + def _diff_versions(before: dict[str, str], after: dict[str, str]) -> Sequence[UpdateInfo]: """diff to parsed pipenv lock files""" return [ @@ -83,34 +111,61 @@ def _update_piplock(piplock_path: Path) -> Sequence[UpdateInfo]: return changed_versions -def _commit_piplock(piplock_path: Path, version_diff: Sequence[UpdateInfo]) -> None: +def _update_requirementslock(requirements_lock: Path) -> Sequence[UpdateInfo]: + logging.info("Updating %s", requirements_lock) + before_versions = _parse_requirements_lock(requirements_lock) subprocess.run( - ["git", "add", "Pipfile.lock"], - cwd=piplock_path.parent, + ["bazel", "run", ":requirements.update"], + cwd=requirements_lock.parent, check=True, + env=environment, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + after_versions = _parse_requirements_lock(requirements_lock) + changed_versions = _diff_versions(before_versions, after_versions) + for info in changed_versions: + logging.info("%s: %s -> %s", info.name, info.from_version, info.to_version) + return changed_versions - message = f"pipenv lock{"" if (p := piplock_path.parent) == Path() else f" in {p}"}\n\n" - message += "\n".join( - f"{info.name}: {info.from_version} -> {info.to_version}" for info in version_diff - ) + +def _commit_lock(commit_infos: Sequence[CommitInfo]) -> None: + message = f"lock python dependencies {"" if (p := commit_infos[0].file_to_be_committed.parent) == Path() else f" in {p}"}\n\n" + for ci in commit_infos: + subprocess.run( + ["git", "add", ci.file_to_be_committed], + cwd=ci.file_to_be_committed.parent, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + message += f"\n{ci.file_to_be_committed.name} update:\n" + message += "\n".join( + f"* {info.name}: {info.from_version} -> {info.to_version}" for info in ci.update_info + ) message += "\n" subprocess.run( ["git", "commit", "-m", message], - cwd=piplock_path.parent, + cwd=commit_infos[0].file_to_be_committed.parent, check=True, ) -def _handle_piplock(task_queue: queue.Queue[Path], lock: threading.Lock) -> None: +def _handle_lock_files(task_queue: queue.Queue[LockPaths], lock: threading.Lock) -> None: while True: - piplock_path = task_queue.get() - if version_diff := _update_piplock(piplock_path): + lock_files = task_queue.get() + commit_infos = [] + if lock_files.requirement_lock is not None and ( + version_diff_req := _update_requirementslock(lock_files.requirement_lock) + ): + commit_infos.append(CommitInfo(lock_files.requirement_lock, version_diff_req)) + if version_diff_pipfile := _update_piplock(lock_files.pipfile_lock): + commit_infos.append(CommitInfo(lock_files.pipfile_lock, version_diff_pipfile)) + if commit_infos: with lock: # only one thread should commit at a time - _commit_piplock(piplock_path, version_diff) + _commit_lock(commit_infos) task_queue.task_done() @@ -120,12 +175,12 @@ def _handle_piplock(task_queue: queue.Queue[Path], lock: threading.Lock) -> None logging.basicConfig(level=logging.INFO) commit_lock = threading.Lock() - tasks = queue.Queue[Path]() + tasks = queue.Queue[LockPaths]() for _ in range(N_THREADS): - threading.Thread(target=_handle_piplock, args=(tasks, commit_lock), daemon=True).start() + threading.Thread(target=_handle_lock_files, args=(tasks, commit_lock), daemon=True).start() - for path in _find_pipfile_locks(): - tasks.put(path) + for lock_paths in _find_locks(): + tasks.put(lock_paths) tasks.join()