diff --git a/.gitignore b/.gitignore index 231f6e2..52b09af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,14 @@ devel/downloads/packages/* toolchain/packages/* install_toolchain.sh + +# python generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# venv +.venv diff --git a/devel/downloads/check_updates/.python-version b/devel/downloads/check_updates/.python-version new file mode 100644 index 0000000..8531a3b --- /dev/null +++ b/devel/downloads/check_updates/.python-version @@ -0,0 +1 @@ +3.12.2 diff --git a/devel/downloads/check_updates/README.md b/devel/downloads/check_updates/README.md new file mode 100644 index 0000000..5464172 --- /dev/null +++ b/devel/downloads/check_updates/README.md @@ -0,0 +1,3 @@ +# check-updates + +Check updates for packages diff --git a/devel/downloads/check_updates/data/packages.sh b/devel/downloads/check_updates/data/packages.sh new file mode 120000 index 0000000..3fb7ba2 --- /dev/null +++ b/devel/downloads/check_updates/data/packages.sh @@ -0,0 +1 @@ +../../packages.sh \ No newline at end of file diff --git a/devel/downloads/check_updates/pyproject.toml b/devel/downloads/check_updates/pyproject.toml new file mode 100644 index 0000000..8dcb3a4 --- /dev/null +++ b/devel/downloads/check_updates/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "check-updates" +version = "0.1.0" +description = "Check updates for packages" +authors = [ + { name = "Adonis Ling", email = "adonis0147@gmail.com" } +] +dependencies = [ + "packaging>=23.2", + "requests>=2.31.0", +] +readme = "README.md" +requires-python = ">= 3.8" + +[project.scripts] +check_updates = "check_updates:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [ + "python-lsp-server[all]>=1.10.0", + "pylsp-mypy>=0.6.8", + "types-requests>=2.31.0.20240218", + "python-lsp-ruff>=2.1.0", + "python-lsp-isort>=0.1", +] + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/check_updates"] diff --git a/devel/downloads/check_updates/requirements-dev.lock b/devel/downloads/check_updates/requirements-dev.lock new file mode 100644 index 0000000..42908c3 --- /dev/null +++ b/devel/downloads/check_updates/requirements-dev.lock @@ -0,0 +1,113 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false + +--trusted-host mirrors.aliyun.com +-e file:. +astroid==3.0.3 + # via pylint +attrs==23.2.0 + # via cattrs + # via lsprotocol +autopep8==2.0.4 + # via python-lsp-server +cattrs==23.2.3 + # via lsprotocol + # via python-lsp-ruff +certifi==2024.2.2 + # via requests +charset-normalizer==3.3.2 + # via requests +dill==0.3.8 + # via pylint +docstring-to-markdown==0.15 + # via python-lsp-server +flake8==7.0.0 + # via python-lsp-server +idna==3.6 + # via requests +importlib-metadata==7.0.1 + # via yapf +isort==5.13.2 + # via pylint + # via python-lsp-isort +jedi==0.19.1 + # via python-lsp-server +lsprotocol==2023.0.1 + # via python-lsp-ruff +mccabe==0.7.0 + # via flake8 + # via pylint + # via python-lsp-server +mypy==1.8.0 + # via pylsp-mypy +mypy-extensions==1.0.0 + # via mypy +packaging==23.2 + # via check-updates + # via pytoolconfig +parso==0.8.3 + # via jedi +platformdirs==4.2.0 + # via pylint + # via pytoolconfig + # via yapf +pluggy==1.4.0 + # via python-lsp-server +pycodestyle==2.11.1 + # via autopep8 + # via flake8 + # via python-lsp-server +pydocstyle==6.3.0 + # via python-lsp-server +pyflakes==3.2.0 + # via flake8 + # via python-lsp-server +pylint==3.0.4 + # via python-lsp-server +pylsp-mypy==0.6.8 +python-lsp-isort==0.1 +python-lsp-jsonrpc==1.1.2 + # via python-lsp-server +python-lsp-ruff==2.1.0 +python-lsp-server==1.10.0 + # via pylsp-mypy + # via python-lsp-isort + # via python-lsp-ruff + # via python-lsp-server +pytoolconfig==1.3.1 + # via pytoolconfig + # via rope +requests==2.31.0 + # via check-updates +rope==1.12.0 + # via python-lsp-server +ruff==0.2.2 + # via python-lsp-ruff +snowballstemmer==2.2.0 + # via pydocstyle +tomli==2.0.1 + # via pylsp-mypy + # via yapf +tomlkit==0.12.4 + # via pylint +types-requests==2.31.0.20240218 +typing-extensions==4.10.0 + # via mypy +ujson==5.9.0 + # via python-lsp-jsonrpc + # via python-lsp-server +urllib3==2.2.1 + # via requests + # via types-requests +whatthepatch==1.0.5 + # via python-lsp-server +yapf==0.40.2 + # via python-lsp-server +zipp==3.17.0 + # via importlib-metadata diff --git a/devel/downloads/check_updates/requirements.lock b/devel/downloads/check_updates/requirements.lock new file mode 100644 index 0000000..9ea5ffb --- /dev/null +++ b/devel/downloads/check_updates/requirements.lock @@ -0,0 +1,23 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false + +--trusted-host mirrors.aliyun.com +-e file:. +certifi==2024.2.2 + # via requests +charset-normalizer==3.3.2 + # via requests +idna==3.6 + # via requests +packaging==23.2 + # via check-updates +requests==2.31.0 + # via check-updates +urllib3==2.2.1 + # via requests diff --git a/devel/downloads/check_updates/src/check_updates/__init__.py b/devel/downloads/check_updates/src/check_updates/__init__.py new file mode 100644 index 0000000..9772803 --- /dev/null +++ b/devel/downloads/check_updates/src/check_updates/__init__.py @@ -0,0 +1,3 @@ +from . import check_updates + +main = check_updates.main diff --git a/devel/downloads/check_updates/src/check_updates/__main__.py b/devel/downloads/check_updates/src/check_updates/__main__.py new file mode 100644 index 0000000..da67e19 --- /dev/null +++ b/devel/downloads/check_updates/src/check_updates/__main__.py @@ -0,0 +1,4 @@ +import check_updates +import sys + +sys.exit(check_updates.main()) \ No newline at end of file diff --git a/devel/downloads/check_updates.py b/devel/downloads/check_updates/src/check_updates/check_updates.py old mode 100755 new mode 100644 similarity index 53% rename from devel/downloads/check_updates.py rename to devel/downloads/check_updates/src/check_updates/check_updates.py index 57754b3..06ede91 --- a/devel/downloads/check_updates.py +++ b/devel/downloads/check_updates/src/check_updates/check_updates.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import json import logging import os @@ -25,28 +23,27 @@ def get(*args, **kwargs): @singleton class GNUFTP: def __init__(self): - self.ftp = FTP('ftp.gnu.org') + self.ftp = FTP("ftp.gnu.org") self.ftp.login() class GitHubGraphQL: - def __init__(self): - self.url = 'https://api.github.com/graphql' - self.token = os.getenv('GITHUB_TOKEN') + self.url = "https://api.github.com/graphql" + self.token = os.getenv("GITHUB_TOKEN") def request(self, query: str, variables: dict): payload = { - 'query': textwrap.dedent(query).strip(), - 'variables': variables, + "query": textwrap.dedent(query).strip(), + "variables": variables, } - headers = {'Authorization': 'Bearer {}'.format(self.token)} + headers = {"Authorization": "Bearer {}".format(self.token)} response = requests.post(self.url, data=json.dumps(payload), headers=headers) response.raise_for_status() return response.json() def get_latest_version(self, owner: str, name: str) -> str: - query = ''' + query = """ query GetLatestTag($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { refs(refPrefix: "refs/tags/", orderBy: { field: TAG_COMMIT_DATE, direction: DESC }, first: 1) { @@ -56,61 +53,66 @@ def get_latest_version(self, owner: str, name: str) -> str: } } } - ''' - variables = {'owner': owner, 'name': name} + """ + variables = {"owner": owner, "name": name} response = self.request(query, variables) - if 'errors' in response: - raise RuntimeError(response['errors']) - return response['data']['repository']['refs']['nodes'][0]['name'] + if "errors" in response: + raise RuntimeError(response["errors"]) + return response["data"]["repository"]["refs"]["nodes"][0]["name"] def get_all_urls(file: str) -> dict[str, str]: urls = dict() - with open(file, 'r') as f: - for line in filter(lambda line: '_URL=' in line, f): - package = line[:line.find('_PACKAGE_URL')] - url = line[line.find('=') + 1:].strip("'\n\"") + with open(file, "r") as f: + for line in filter(lambda line: "_URL=" in line, f): + package = line[: line.find("_PACKAGE_URL")] + url = line[line.find("=") + 1 :].strip("'\n\"") urls[package] = url return urls def check_gnu_package(package: str, url: str) -> bool: ftp = GNUFTP().ftp - ftp.cwd('/gnu/{}'.format(package.lower())) - name = package.lower() if package != 'WGET' else 'wget2' + ftp.cwd("/gnu/{}".format(package.lower())) + name = package.lower() if package != "WGET" else "wget2" def get_version(file): - match = re.compile(r'{}-(\d+(\.\d+)*).*'.format(name)).match(file) + match = re.compile(r"{}-(\d+(\.\d+)*).*".format(name)).match(file) assert match is not None return match.group(1) versions = [ get_version(file) - for file in ftp.nlst() if re.compile(r'{}-(\d+(\.\d+)*).*\.tar\.gz'.format(name)).match(file) + for file in ftp.nlst() + if re.compile(r"{}-(\d+(\.\d+)*).*\.tar\.gz".format(name)).match(file) ] versions.sort(key=packaging.version.parse) latest_version = versions[-1] file = os.path.basename(url) - version = file[len(name) + 1:-len('.tar.gz')] - logging.log(logging.INFO if version == latest_version else logging.WARN, - 'Check {}: {} -> {}'.format(package, version, latest_version)) + version = file[len(name) + 1 : -len(".tar.gz")] + logging.log( + logging.INFO if version == latest_version else logging.WARN, + "Check {}: {} -> {}".format(package, version, latest_version), + ) return version != latest_version def check_github_package(package: str, url: str) -> bool: - match = re.compile(r'https://github.com/([^/]*)/([^/]*)/.*').match(url) + match = re.compile(r"https://github.com/([^/]*)/([^/]*)/.*").match(url) assert match is not None owner, name = match.group(1), match.group(2) latest_version = GitHubGraphQL().get_latest_version(owner, name) - if '/refs/tags/' in url: - version = os.path.basename(url)[:-len('.tar.gz')] - elif '/releases/download/' in url: - match = re.compile(r'.*/releases/download/([^/]+)/.*').match(url) + if "/refs/tags/" in url: + version = os.path.basename(url)[: -len(".tar.gz")] + elif "/releases/download/" in url: + match = re.compile(r".*/releases/download/([^/]+)/.*").match(url) assert match is not None version = match.group(1) - logging.log(logging.INFO if version == latest_version else logging.WARN, - 'Check {}: {} -> {}'.format(package, version, latest_version)) + logging.log( + logging.INFO if version == latest_version else logging.WARN, + "Check {}: {} -> {}".format(package, version, latest_version), + ) return version != latest_version @@ -118,29 +120,35 @@ def check_updates(urls: dict[str, str]) -> tuple[list[str], list[str]]: checked = [] updates = [] for package, url in urls.items(): - if 'ftpmirror.gnu.org' in url: + if "ftpmirror.gnu.org" in url: if check_gnu_package(package, url): updates.append(package) checked.append(package) - elif 'github.com' in url: + elif "github.com" in url: if check_github_package(package, url): updates.append(package) checked.append(package) return (updates, checked) -def main(): - logging.basicConfig(level=logging.INFO, - format='[%(levelname)s] %(asctime)s - (%(filename)s:%(lineno)d) - %(message)s') +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="[%(levelname)s] %(asctime)s - (%(filename)s:%(lineno)d) - %(message)s", + ) - urls = get_all_urls('{}/packages.sh'.format(os.path.dirname(__file__))) + urls = get_all_urls( + "{}/packages.sh".format( + os.path.join(os.path.dirname(__file__), "..", "..", "data") + ) + ) updates, checked = check_updates(urls) - logging.info('These packages should be updated: {}'.format(' '.join(updates))) + logging.info("These packages should be updated: {}".format(" ".join(updates))) for package in checked: urls.pop(package) - logging.info('These packages should be checked manually: {}'.format(' '.join(urls.keys()))) - + logging.info( + "These packages should be checked manually: {}".format(" ".join(urls.keys())) + ) -if __name__ == '__main__': - main() + return 0