From f47fc132f383a7faa2f5e57a504fd509b43faaf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Pi=C3=A9dallu?= Date: Wed, 28 Feb 2024 00:24:10 +0100 Subject: [PATCH] Add 'added_date' and 'killed_date' to catalog files. All a tool save_added_date.py that can be used to easily update the date. --- .github/workflows/added_dates.yml | 29 ++++++ schemas/apps.toml.schema.json | 6 ++ schemas/graveyard.toml.schema.json | 8 +- schemas/wishlist.toml.schema.json | 5 +- tools/list_builder.py | 38 +------- tools/save_added_date.py | 137 +++++++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/added_dates.yml create mode 100755 tools/save_added_date.py diff --git a/.github/workflows/added_dates.yml b/.github/workflows/added_dates.yml new file mode 100644 index 0000000000..6fc2ba367f --- /dev/null +++ b/.github/workflows/added_dates.yml @@ -0,0 +1,29 @@ +name: Add Added dates to the catalog files + +on: + push: + pull_request_target: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Install toml python lib + run: | + pip3 install toml tomlkit gitpython + - name: Add added dates to catalog files + run: | + ./tools/save_added_date.py + + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "Automatically add dates to the catalog files" + file_pattern: 'apps.toml wishlist.toml graveyard.toml' diff --git a/schemas/apps.toml.schema.json b/schemas/apps.toml.schema.json index 4833c35d60..42e15f519b 100644 --- a/schemas/apps.toml.schema.json +++ b/schemas/apps.toml.schema.json @@ -56,6 +56,12 @@ }, "branch": { "type": "string" + }, + "added_date": { + "type": "integer" + }, + "deprecated_date": { + "type": "integer" } } } diff --git a/schemas/graveyard.toml.schema.json b/schemas/graveyard.toml.schema.json index ec22f8c86a..847c548520 100644 --- a/schemas/graveyard.toml.schema.json +++ b/schemas/graveyard.toml.schema.json @@ -41,8 +41,14 @@ "type": "string" }, "additionalItems": false + }, + "killed_date": { + "type": "integer" + }, + "deprecated_date": { + "type": "integer" } } } } -} \ No newline at end of file +} diff --git a/schemas/wishlist.toml.schema.json b/schemas/wishlist.toml.schema.json index b42caac108..6cac0e688d 100644 --- a/schemas/wishlist.toml.schema.json +++ b/schemas/wishlist.toml.schema.json @@ -32,8 +32,11 @@ "draft": { "type": "string", "format": "url" + }, + "added_date": { + "type": "integer" } } } } -} \ No newline at end of file +} diff --git a/tools/list_builder.py b/tools/list_builder.py index b8c4976007..bad1d32921 100755 --- a/tools/list_builder.py +++ b/tools/list_builder.py @@ -193,42 +193,8 @@ def build_app_dict(app, infos): repo = Repo(this_app_cache) - commits_in_apps_json = ( - Repo(REPO_APPS_ROOT) - .git.log( - "-S", - f'"{app}"', - "--first-parent", - "--reverse", - "--date=unix", - "--format=%cd", - "--", - "apps.json", - ) - .split("\n") - ) - if len(commits_in_apps_json) > 1: - first_commit = commits_in_apps_json[0] - else: - commits_in_apps_toml = ( - Repo(REPO_APPS_ROOT) - .git.log( - "-S", - f"[{app}]", - "--first-parent", - "--reverse", - "--date=unix", - "--format=%cd", - "--", - "apps.json", - "apps.toml", - ) - .split("\n") - ) - first_commit = commits_in_apps_toml[0] - - # Assume the first entry we get (= the oldest) is the time the app was added - infos["added_in_catalog"] = int(first_commit) + # If added_date is not present, we are in a github action of the PR that adds it... so default to a bad value. + infos["added_in_catalog"] = infos.get("added_date", 0) # int(commit_timestamps_for_this_app_in_catalog.split("\n")[0]) infos["branch"] = infos.get("branch", "master") diff --git a/tools/save_added_date.py b/tools/save_added_date.py new file mode 100755 index 0000000000..1d28d93f8c --- /dev/null +++ b/tools/save_added_date.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import tomlkit +import json +from datetime import datetime +from git import Repo, Commit +from pathlib import Path +import logging +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + REPO_APPS_ROOT = Path() +else: + from appslib.utils import REPO_APPS_ROOT + + +def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None: + repo = Repo(repo_path) + + # Start with whole repo + first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0) + repo.git.bisect("reset") + repo.git.bisect("start", "--no-checkout", "HEAD", first_commit) + + while True: + try: + status = "bad" if is_newer(repo.commit("BISECT_HEAD")) else "good" + except Exception: + status = "skip" + result_string = repo.git.bisect(status) + if "is the first bad commit" in result_string.splitlines()[0]: + return repo.commit(result_string.splitlines()[0].split(" ", 1)[0]) + + +def get_app_info(commit: Commit, filebase: str, name: str) -> dict | None: + data = None + try: + filestream = commit.tree.join(f"{filebase}.toml") + filedata = filestream.data_stream.read().decode("utf-8") + dictdata = tomlkit.loads(filedata) + data = dictdata[name] + except KeyError: + pass + try: + filestream = commit.tree.join(f"{filebase}.json") + filedata = filestream.data_stream.read().decode("utf-8") + dictdata = json.loads(filedata) + data = dictdata[name] + except KeyError: + pass + + assert isinstance(data, dict) or data is None + return data + + +def app_is_present(commit: Commit, name: str) -> bool: + info = get_app_info(commit, "apps", name) + # if info is None: + # info = get_app_info(commit, "graveyard", name) + return info is not None + + +def app_is_deprecated(commit: Commit, name: str) -> bool: + info = get_app_info(commit, "apps", name) + if info is None: + return False + + antifeatures = info.get("antifeatures", []) + return "deprecated-software" in antifeatures + + +def date_added(name: str) -> int | None: + result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name)) + print(result) + return None if result is None else result.committed_date + + +def date_deprecated(name: str) -> int | None: + result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name)) + print(result) + return None if result is None else result.committed_date + + +def add_deprecation_dates(file: Path) -> None: + key = "deprecated_date" + document = tomlkit.load(file.open("r", encoding="utf-8")) + for app, info in document.items(): + if key in info.keys(): + continue + if "deprecated-software" not in info.get("antifeatures", []): + continue + date = date_deprecated(app) + if date is None: + continue + info[key] = date + info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d")) + info[key].trivia.comment_ws = " " + tomlkit.dump(document, file.open("w")) + + +def date_added_to(match: str, file: Path) -> int | None: + commits = Repo(REPO_APPS_ROOT).git.log( + "-S", match, "--first-parent", "--reverse", + "--date=unix", "--format=%cd", "--", str(file)).splitlines() + + if not commits: + return None + first_commit = commits[0] + return int(first_commit) + + +def add_apparition_dates(file: Path, key: str) -> None: + document = tomlkit.load(file.open("r", encoding="utf-8")) + for app, info in document.items(): + if key in info.keys(): + continue + date = date_added_to(f"[{app}]", file) + assert date is not None + info[key] = date + info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d")) + info[key].trivia.comment_ws = " " + tomlkit.dump(document, file.open("w")) + + +def main() -> None: + logging.basicConfig(level=logging.DEBUG) + + add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date") + add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date") + add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date") + + add_deprecation_dates(REPO_APPS_ROOT/"apps.toml") + add_deprecation_dates(REPO_APPS_ROOT/"graveyard.toml") + + +if __name__ == "__main__": + main()