Refactor check_updates.py

.gitignore

@@ -1,3 +1,14 @@
 devel/downloads/packages/*
 toolchain/packages/*
 install_toolchain.sh
+
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv

devel/downloads/check_updates/.python-version

@@ -0,0 +1 @@
+3.12.2

devel/downloads/check_updates/README.md

@@ -0,0 +1,3 @@
+# check-updates
+
+Check updates for packages

devel/downloads/check_updates/data/packages.sh

@@ -0,0 +1 @@
+../../packages.sh

devel/downloads/check_updates/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "check-updates"
+version = "0.1.0"
+description = "Check updates for packages"
+authors = [
+    { name = "Adonis Ling", email = "[email protected]" }
+]
+dependencies = [
+    "packaging>=23.2",
+    "requests>=2.31.0",
+]
+readme = "README.md"
+requires-python = ">= 3.8"
+
+[project.scripts]
+check_updates = "check_updates:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "python-lsp-server[all]>=1.10.0",
+    "pylsp-mypy>=0.6.8",
+    "types-requests>=2.31.0.20240218",
+    "python-lsp-ruff>=2.1.0",
+    "python-lsp-isort>=0.1",
+]
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/check_updates"]

devel/downloads/check_updates/requirements-dev.lock

@@ -0,0 +1,113 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+
+--trusted-host mirrors.aliyun.com
+-e file:.
+astroid==3.0.3
+    # via pylint
+attrs==23.2.0
+    # via cattrs
+    # via lsprotocol
+autopep8==2.0.4
+    # via python-lsp-server
+cattrs==23.2.3
+    # via lsprotocol
+    # via python-lsp-ruff
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+dill==0.3.8
+    # via pylint
+docstring-to-markdown==0.15
+    # via python-lsp-server
+flake8==7.0.0
+    # via python-lsp-server
+idna==3.6
+    # via requests
+importlib-metadata==7.0.1
+    # via yapf
+isort==5.13.2
+    # via pylint
+    # via python-lsp-isort
+jedi==0.19.1
+    # via python-lsp-server
+lsprotocol==2023.0.1
+    # via python-lsp-ruff
+mccabe==0.7.0
+    # via flake8
+    # via pylint
+    # via python-lsp-server
+mypy==1.8.0
+    # via pylsp-mypy
+mypy-extensions==1.0.0
+    # via mypy
+packaging==23.2
+    # via check-updates
+    # via pytoolconfig
+parso==0.8.3
+    # via jedi
+platformdirs==4.2.0
+    # via pylint
+    # via pytoolconfig
+    # via yapf
+pluggy==1.4.0
+    # via python-lsp-server
+pycodestyle==2.11.1
+    # via autopep8
+    # via flake8
+    # via python-lsp-server
+pydocstyle==6.3.0
+    # via python-lsp-server
+pyflakes==3.2.0
+    # via flake8
+    # via python-lsp-server
+pylint==3.0.4
+    # via python-lsp-server
+pylsp-mypy==0.6.8
+python-lsp-isort==0.1
+python-lsp-jsonrpc==1.1.2
+    # via python-lsp-server
+python-lsp-ruff==2.1.0
+python-lsp-server==1.10.0
+    # via pylsp-mypy
+    # via python-lsp-isort
+    # via python-lsp-ruff
+    # via python-lsp-server
+pytoolconfig==1.3.1
+    # via pytoolconfig
+    # via rope
+requests==2.31.0
+    # via check-updates
+rope==1.12.0
+    # via python-lsp-server
+ruff==0.2.2
+    # via python-lsp-ruff
+snowballstemmer==2.2.0
+    # via pydocstyle
+tomli==2.0.1
+    # via pylsp-mypy
+    # via yapf
+tomlkit==0.12.4
+    # via pylint
+types-requests==2.31.0.20240218
+typing-extensions==4.10.0
+    # via mypy
+ujson==5.9.0
+    # via python-lsp-jsonrpc
+    # via python-lsp-server
+urllib3==2.2.1
+    # via requests
+    # via types-requests
+whatthepatch==1.0.5
+    # via python-lsp-server
+yapf==0.40.2
+    # via python-lsp-server
+zipp==3.17.0
+    # via importlib-metadata

devel/downloads/check_updates/requirements.lock

@@ -0,0 +1,23 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+
+--trusted-host mirrors.aliyun.com
+-e file:.
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+idna==3.6
+    # via requests
+packaging==23.2
+    # via check-updates
+requests==2.31.0
+    # via check-updates
+urllib3==2.2.1
+    # via requests

devel/downloads/check_updates/src/check_updates/__init__.py

@@ -0,0 +1,3 @@
+from . import check_updates
+
+main = check_updates.main

devel/downloads/check_updates/src/check_updates/__main__.py

@@ -0,0 +1,4 @@
+import check_updates
+import sys
+
+sys.exit(check_updates.main())

devel/downloads/check_updates.py → devel/downloads/check_updates/src/check_updates/check_updates.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
 import json
 import logging
 import os
@@ -25,28 +23,27 @@ def get(*args, **kwargs):
 @singleton
 class GNUFTP:
     def __init__(self):
-        self.ftp = FTP('ftp.gnu.org')
+        self.ftp = FTP("ftp.gnu.org")
         self.ftp.login()
 
 
 class GitHubGraphQL:
-
     def __init__(self):
-        self.url = 'https://api.github.com/graphql'
-        self.token = os.getenv('GITHUB_TOKEN')
+        self.url = "https://api.github.com/graphql"
+        self.token = os.getenv("GITHUB_TOKEN")
 
     def request(self, query: str, variables: dict):
         payload = {
-            'query': textwrap.dedent(query).strip(),
-            'variables': variables,
+            "query": textwrap.dedent(query).strip(),
+            "variables": variables,
         }
-        headers = {'Authorization': 'Bearer {}'.format(self.token)}
+        headers = {"Authorization": "Bearer {}".format(self.token)}
         response = requests.post(self.url, data=json.dumps(payload), headers=headers)
         response.raise_for_status()
         return response.json()
 
     def get_latest_version(self, owner: str, name: str) -> str:
-        query = '''
+        query = """
             query GetLatestTag($owner: String!, $name: String!) {
                 repository(owner: $owner, name: $name) {
                     refs(refPrefix: "refs/tags/", orderBy: { field: TAG_COMMIT_DATE, direction: DESC }, first: 1) {
@@ -56,91 +53,102 @@ def get_latest_version(self, owner: str, name: str) -> str:
                     }
                 }
             }
-        '''
-        variables = {'owner': owner, 'name': name}
+        """
+        variables = {"owner": owner, "name": name}
         response = self.request(query, variables)
-        if 'errors' in response:
-            raise RuntimeError(response['errors'])
-        return response['data']['repository']['refs']['nodes'][0]['name']
+        if "errors" in response:
+            raise RuntimeError(response["errors"])
+        return response["data"]["repository"]["refs"]["nodes"][0]["name"]
 
 
 def get_all_urls(file: str) -> dict[str, str]:
     urls = dict()
-    with open(file, 'r') as f:
-        for line in filter(lambda line: '_URL=' in line, f):
-            package = line[:line.find('_PACKAGE_URL')]
-            url = line[line.find('=') + 1:].strip("'\n\"")
+    with open(file, "r") as f:
+        for line in filter(lambda line: "_URL=" in line, f):
+            package = line[: line.find("_PACKAGE_URL")]
+            url = line[line.find("=") + 1 :].strip("'\n\"")
             urls[package] = url
     return urls
 
 
 def check_gnu_package(package: str, url: str) -> bool:
     ftp = GNUFTP().ftp
-    ftp.cwd('/gnu/{}'.format(package.lower()))
-    name = package.lower() if package != 'WGET' else 'wget2'
+    ftp.cwd("/gnu/{}".format(package.lower()))
+    name = package.lower() if package != "WGET" else "wget2"
 
     def get_version(file):
-        match = re.compile(r'{}-(\d+(\.\d+)*).*'.format(name)).match(file)
+        match = re.compile(r"{}-(\d+(\.\d+)*).*".format(name)).match(file)
         assert match is not None
         return match.group(1)
 
     versions = [
         get_version(file)
-        for file in ftp.nlst() if re.compile(r'{}-(\d+(\.\d+)*).*\.tar\.gz'.format(name)).match(file)
+        for file in ftp.nlst()
+        if re.compile(r"{}-(\d+(\.\d+)*).*\.tar\.gz".format(name)).match(file)
     ]
     versions.sort(key=packaging.version.parse)
     latest_version = versions[-1]
     file = os.path.basename(url)
-    version = file[len(name) + 1:-len('.tar.gz')]
-    logging.log(logging.INFO if version == latest_version else logging.WARN,
-                'Check {}: {} -> {}'.format(package, version, latest_version))
+    version = file[len(name) + 1 : -len(".tar.gz")]
+    logging.log(
+        logging.INFO if version == latest_version else logging.WARN,
+        "Check {}: {} -> {}".format(package, version, latest_version),
+    )
     return version != latest_version
 
 
 def check_github_package(package: str, url: str) -> bool:
-    match = re.compile(r'https://github.com/([^/]*)/([^/]*)/.*').match(url)
+    match = re.compile(r"https://github.com/([^/]*)/([^/]*)/.*").match(url)
     assert match is not None
     owner, name = match.group(1), match.group(2)
 
     latest_version = GitHubGraphQL().get_latest_version(owner, name)
-    if '/refs/tags/' in url:
-        version = os.path.basename(url)[:-len('.tar.gz')]
-    elif '/releases/download/' in url:
-        match = re.compile(r'.*/releases/download/([^/]+)/.*').match(url)
+    if "/refs/tags/" in url:
+        version = os.path.basename(url)[: -len(".tar.gz")]
+    elif "/releases/download/" in url:
+        match = re.compile(r".*/releases/download/([^/]+)/.*").match(url)
         assert match is not None
         version = match.group(1)
-    logging.log(logging.INFO if version == latest_version else logging.WARN,
-                'Check {}: {} -> {}'.format(package, version, latest_version))
+    logging.log(
+        logging.INFO if version == latest_version else logging.WARN,
+        "Check {}: {} -> {}".format(package, version, latest_version),
+    )
     return version != latest_version
 
 
 def check_updates(urls: dict[str, str]) -> tuple[list[str], list[str]]:
     checked = []
     updates = []
     for package, url in urls.items():
-        if 'ftpmirror.gnu.org' in url:
+        if "ftpmirror.gnu.org" in url:
             if check_gnu_package(package, url):
                 updates.append(package)
             checked.append(package)
-        elif 'github.com' in url:
+        elif "github.com" in url:
             if check_github_package(package, url):
                 updates.append(package)
             checked.append(package)
     return (updates, checked)
 
 
-def main():
-    logging.basicConfig(level=logging.INFO,
-                        format='[%(levelname)s] %(asctime)s - (%(filename)s:%(lineno)d) - %(message)s')
+def main() -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="[%(levelname)s] %(asctime)s - (%(filename)s:%(lineno)d) - %(message)s",
+    )
 
-    urls = get_all_urls('{}/packages.sh'.format(os.path.dirname(__file__)))
+    urls = get_all_urls(
+        "{}/packages.sh".format(
+            os.path.join(os.path.dirname(__file__), "..", "..", "data")
+        )
+    )
     updates, checked = check_updates(urls)
 
-    logging.info('These packages should be updated: {}'.format(' '.join(updates)))
+    logging.info("These packages should be updated: {}".format(" ".join(updates)))
     for package in checked:
         urls.pop(package)
-    logging.info('These packages should be checked manually: {}'.format(' '.join(urls.keys())))
-
+    logging.info(
+        "These packages should be checked manually: {}".format(" ".join(urls.keys()))
+    )
 
-if __name__ == '__main__':
-    main()
+    return 0