Add 'added_date' and 'killed_date' to catalog files. All a tool save_…

…added_date.py that can be used to easily update the date.

.github/workflows/added_dates.yml

@@ -0,0 +1,29 @@
+name: Add Added dates to the catalog files
+
+on:
+  push:
+  pull_request_target:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - name: Install toml python lib
+      run: |
+        pip3 install toml tomlkit gitpython
+    - name: Add added dates to catalog files
+      run: |
+        ./tools/save_added_date.py
+
+    - uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: "Automatically add dates to the catalog files"
+        file_pattern: 'apps.toml wishlist.toml graveyard.toml'

schemas/apps.toml.schema.json

@@ -56,6 +56,12 @@
                 },
                 "branch": {
                     "type": "string"
+                },
+                "added_date": {
+                    "type": "integer"
+                },
+                "deprecated_date": {
+                    "type": "integer"
                 }
             }
         }

schemas/graveyard.toml.schema.json

@@ -41,8 +41,14 @@
                         "type": "string"
                     },
                     "additionalItems": false
+                },
+                "killed_date": {
+                    "type": "integer"
+                },
+                "deprecated_date": {
+                    "type": "integer"
                 }
             }
         }
     }
-}
+}

schemas/wishlist.toml.schema.json

@@ -32,8 +32,11 @@
                 "draft": {
                     "type": "string",
                     "format": "url"
+                },
+                "added_date": {
+                    "type": "integer"
                 }
             }
         }
     }
-}
+}

tools/list_builder.py

@@ -193,42 +193,8 @@ def build_app_dict(app, infos):
 
     repo = Repo(this_app_cache)
 
-    commits_in_apps_json = (
-        Repo(REPO_APPS_ROOT)
-        .git.log(
-            "-S",
-            f'"{app}"',
-            "--first-parent",
-            "--reverse",
-            "--date=unix",
-            "--format=%cd",
-            "--",
-            "apps.json",
-        )
-        .split("\n")
-    )
-    if len(commits_in_apps_json) > 1:
-        first_commit = commits_in_apps_json[0]
-    else:
-        commits_in_apps_toml = (
-            Repo(REPO_APPS_ROOT)
-            .git.log(
-                "-S",
-                f"[{app}]",
-                "--first-parent",
-                "--reverse",
-                "--date=unix",
-                "--format=%cd",
-                "--",
-                "apps.json",
-                "apps.toml",
-            )
-            .split("\n")
-        )
-        first_commit = commits_in_apps_toml[0]
-
-    # Assume the first entry we get (= the oldest) is the time the app was added
-    infos["added_in_catalog"] = int(first_commit)
+    # If added_date is not present, we are in a github action of the PR that adds it... so default to a bad value.
+    infos["added_in_catalog"] = infos.get("added_date", 0)
     # int(commit_timestamps_for_this_app_in_catalog.split("\n")[0])
 
     infos["branch"] = infos.get("branch", "master")

tools/save_added_date.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+import tomlkit
+import json
+from datetime import datetime
+from git import Repo, Commit
+from pathlib import Path
+import logging
+from typing import TYPE_CHECKING, Callable
+
+if TYPE_CHECKING:
+    REPO_APPS_ROOT = Path()
+else:
+    from appslib.utils import REPO_APPS_ROOT
+
+
+def git_bisect(repo_path: Path, is_newer: Callable[[Commit], bool]) -> Commit | None:
+    repo = Repo(repo_path)
+
+    # Start with whole repo
+    first_commit = repo.git.rev_list("HEAD", reverse=True, max_parents=0)
+    repo.git.bisect("reset")
+    repo.git.bisect("start", "--no-checkout", "HEAD", first_commit)
+
+    while True:
+        try:
+            status = "bad" if is_newer(repo.commit("BISECT_HEAD")) else "good"
+        except Exception:
+            status = "skip"
+        result_string = repo.git.bisect(status)
+        if "is the first bad commit" in result_string.splitlines()[0]:
+            return repo.commit(result_string.splitlines()[0].split(" ", 1)[0])
+
+
+def get_app_info(commit: Commit, filebase: str, name: str) -> dict | None:
+    data = None
+    try:
+        filestream = commit.tree.join(f"{filebase}.toml")
+        filedata = filestream.data_stream.read().decode("utf-8")
+        dictdata = tomlkit.loads(filedata)
+        data = dictdata[name]
+    except KeyError:
+        pass
+    try:
+        filestream = commit.tree.join(f"{filebase}.json")
+        filedata = filestream.data_stream.read().decode("utf-8")
+        dictdata = json.loads(filedata)
+        data = dictdata[name]
+    except KeyError:
+        pass
+
+    assert isinstance(data, dict) or data is None
+    return data
+
+
+def app_is_present(commit: Commit, name: str) -> bool:
+    info = get_app_info(commit, "apps", name)
+    # if info is None:
+    #     info = get_app_info(commit, "graveyard", name)
+    return info is not None
+
+
+def app_is_deprecated(commit: Commit, name: str) -> bool:
+    info = get_app_info(commit, "apps", name)
+    if info is None:
+        return False
+
+    antifeatures = info.get("antifeatures", [])
+    return "deprecated-software" in antifeatures
+
+
+def date_added(name: str) -> int | None:
+    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_present(x, name))
+    print(result)
+    return None if result is None else result.committed_date
+
+
+def date_deprecated(name: str) -> int | None:
+    result = git_bisect(REPO_APPS_ROOT, lambda x: app_is_deprecated(x, name))
+    print(result)
+    return None if result is None else result.committed_date
+
+
+def add_deprecation_dates(file: Path) -> None:
+    key = "deprecated_date"
+    document = tomlkit.load(file.open("r", encoding="utf-8"))
+    for app, info in document.items():
+        if key in info.keys():
+            continue
+        if "deprecated-software" not in info.get("antifeatures", []):
+            continue
+        date = date_deprecated(app)
+        if date is None:
+            continue
+        info[key] = date
+        info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
+        info[key].trivia.comment_ws = "  "
+    tomlkit.dump(document, file.open("w"))
+
+
+def date_added_to(match: str, file: Path) -> int | None:
+    commits = Repo(REPO_APPS_ROOT).git.log(
+        "-S", match, "--first-parent", "--reverse",
+        "--date=unix", "--format=%cd", "--", str(file)).splitlines()
+
+    if not commits:
+        return None
+    first_commit = commits[0]
+    return int(first_commit)
+
+
+def add_apparition_dates(file: Path, key: str) -> None:
+    document = tomlkit.load(file.open("r", encoding="utf-8"))
+    for app, info in document.items():
+        if key in info.keys():
+            continue
+        date = date_added_to(f"[{app}]", file)
+        assert date is not None
+        info[key] = date
+        info[key].comment(datetime.fromtimestamp(info[key]).strftime("%Y/%m/%d"))
+        info[key].trivia.comment_ws = "  "
+    tomlkit.dump(document, file.open("w"))
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.DEBUG)
+
+    add_apparition_dates(REPO_APPS_ROOT / "apps.toml", key="added_date")
+    add_apparition_dates(REPO_APPS_ROOT / "wishlist.toml", key="added_date")
+    add_apparition_dates(REPO_APPS_ROOT / "graveyard.toml", key="killed_date")
+
+    add_deprecation_dates(REPO_APPS_ROOT/"apps.toml")
+    add_deprecation_dates(REPO_APPS_ROOT/"graveyard.toml")
+
+
+if __name__ == "__main__":
+    main()