diff --git a/main.py b/main.py index cd465da..0f3ea6b 100644 --- a/main.py +++ b/main.py @@ -58,7 +58,9 @@ def main(): resource_list += apr_output[0] latest_meta_list += apr_output[1] - resource_list += imagemagick.make_cache() + imagemagick_output = imagemagick.make_cache() + resource_list += imagemagick_output[0] + latest_meta_list.append(imagemagick_output[1]) openresty_output = openresty.make_cache() resource_list += openresty_output[0] @@ -79,23 +81,42 @@ def main(): resource_list += openssl_output[0] latest_meta_list += openssl_output[1] - resource_list += github.download_repo_by_tag("openresty", "lua-resty-core", "tar.gz", True) + lua_resty_core_output = github.download_repo_by_tag("openresty", "lua-resty-core", + "tar.gz", True, "lua_resty_core_ver") + resource_list += lua_resty_core_output[0] + latest_meta_list.append(lua_resty_core_output[1]) + resource_list += pure_ftpd.make_cache() resource_list += htop.make_cache() - resource_list += github.get_single_package_from_release("jemalloc", "jemalloc") - resource_list += github.download_repo_by_tag("openresty", "lua-resty-lrucache", "tar.gz", True) - resource_list += github.download_repo_by_tag("openresty", "luajit2", "tar.gz", True) - resource_list += github.download_repo_by_tag("openresty", "lua-cjson", "tar.gz", True) + + jemalloc_output = github.get_single_package_from_release("jemalloc", "jemalloc", "jemalloc_ver") + resource_list += jemalloc_output[0] + latest_meta_list.append(jemalloc_output[1]) + + lua_resty_lrucache_output = github.download_repo_by_tag("openresty", "lua-resty-lrucache", "tar.gz", True, + "lua_resty_lrucache_ver") + resource_list += lua_resty_lrucache_output[0] + latest_meta_list.append(lua_resty_lrucache_output[1]) + + luajit2_output = github.download_repo_by_tag("openresty", "luajit2", "tar.gz", True, "luajit2_ver") + resource_list += luajit2_output[0] + latest_meta_list.append(luajit2_output[1]) + + lua_cjson_output = github.download_repo_by_tag("openresty", "lua-cjson", "tar.gz", True, "lua_cjson_ver") + resource_list += lua_cjson_output[0] + latest_meta_list.append(lua_cjson_output[1]) + resource_list += github.get_package_from_release_with_regular_expression("gperftools", "gperftools", r"gperftools-\d+.\d+.tar.gz", - 3) + 3, + None)[0] icu_output = github.get_package_from_release_with_regular_expression("unicode-org", "icu", r"(icu4c-)[\d|\-|\_]+(src\.tgz)", 3, - "icu4c") + "icu4c_ver") resource_list += icu_output[0] latest_meta_list.append(icu_output[1]) @@ -103,7 +124,8 @@ def main(): resource_list += github.get_package_from_release_with_regular_expression("glotlabs", "gdrive", r"linux", - 1) + 1, + None)[0] libzip_output = github.get_package_from_release_with_regular_expression("nih-at", "libzip", r"\.tar\.gz", @@ -134,16 +156,17 @@ def main(): resource_list += github.get_package_from_release_with_regular_expression("libevent", "libevent", r"\.tar\.gz$", - 5) - resource_list += github.download_repo_by_tag("jokkedk", "webgrind", "zip", False) + 5, + None)[0] + resource_list += github.download_repo_by_tag("jokkedk", "webgrind", "zip", False, None)[0] # ngx_devel_kit name changed!!! - resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit", "tar.gz", False) + resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit", "tar.gz", False, None)[0] resource_list += github.get_package_from_release_with_regular_expression("kkos", "oniguruma", - r"\.tar\.gz$", 5) + r"\.tar\.gz$", 5, None)[0] resource_list += github.get_package_from_release_with_regular_expression("dropbox", "dbxcli", - r"dbxcli-linux-arm", 1) + r"dbxcli-linux-arm", 1, None)[0] resource_list += github.get_package_from_release_with_regular_expression("dropbox", "dbxcli", - r"dbxcli-linux-amd64", 1) + r"dbxcli-linux-amd64", 1, None)[0] resource_list += bison.make_cache() libiconv_output = libiconv.make_cache() @@ -154,15 +177,61 @@ def main(): resource_list += misc_output[0] latest_meta_list += misc_output[1] - resource_list += php_plugins.make_cache("APCU", "apcu") - resource_list += php_plugins.make_cache("gmagick", "gmagick") - resource_list += php_plugins.make_cache("imagick", "imagick") - resource_list += php_plugins.make_cache("memcache", "memcache") - resource_list += php_plugins.make_cache("mongodb", "mongodb") - resource_list += php_plugins.make_cache("swoole", "swoole") - resource_list += php_plugins.make_cache("YAF", "yaf") - resource_list += php_plugins.make_cache("xdebug", "xdebug") - resource_list += php_plugins.make_cache("mongo", "mongo") + apcu_output = php_plugins.make_cache("APCU", "apcu", + False, "apcu_ver") + resource_list += apcu_output[0] + latest_meta_list.append(apcu_output[1]) + + gmagick_output = php_plugins.make_cache("gmagick", "gmagick", + True, "gmagick_ver") + resource_list += gmagick_output[0] + latest_meta_list.append(gmagick_output[1]) + + imagick_output = php_plugins.make_cache("imagick", "imagick", + False, "imagick_ver") + resource_list += imagick_output[0] + latest_meta_list.append(imagick_output[1]) + + pecl_memcache_output = php_plugins.make_cache("memcache", "memcache", + False, "pecl_memcache_ver") + resource_list += pecl_memcache_output[0] + latest_meta_list.append(pecl_memcache_output[1]) + + pecl_mongodb_output = php_plugins.make_cache("mongodb", "mongodb", + False, "pecl_mongodb_ver") + resource_list += pecl_mongodb_output[0] + latest_meta_list.append(pecl_mongodb_output[1]) + + swoole_output = php_plugins.make_cache("swoole", "swoole", + False, "swoole_ver") + resource_list += swoole_output[0] + latest_meta_list.append(swoole_output[1]) + + yaf_output = php_plugins.make_cache("YAF", "yaf", + False, "yaf_ver") + resource_list += yaf_output[0] + latest_meta_list.append(yaf_output[1]) + + xdebug_output = php_plugins.make_cache("xdebug", "xdebug", + False, "xdebug_ver") + resource_list += xdebug_output[0] + latest_meta_list.append(xdebug_output[1]) + + pecl_mongo_output = php_plugins.make_cache("mongo", "mongo", + False, "pecl_mongo_ver") + resource_list += pecl_mongo_output[0] + latest_meta_list.append(pecl_mongo_output[1]) + + # Older versions of PHP plugins + latest_meta_list += [ + {"version_file_name": "apcu_oldver", "version": "4.0.11"}, + {"version_file_name": "gmagick_oldver", "version": "1.1.7RC3"}, + {"version_file_name": "imagick_oldver", "version": "3.4.4"}, + {"version_file_name": "pecl_memcache_oldver", "version": "4.0.5.2"}, + {"version_file_name": "pecl_mongodb_oldver", "version": "1.9.2"}, + {"version_file_name": "swoole_oldver", "version": "4.8.12"}, + {"version_file_name": "xdebug_oldver", "version": "2.9.8"}, + ] with open(r"./output/resources.json", "w+") as f: f.write(json.dumps(resource_list, indent=4)) @@ -187,9 +256,10 @@ def main(): <h1>Oneinstack Mirror</h1> <p>This page is generated by <a href="https://github.com/dalao-org/oneinstack-mirror-generator">oneinstack-mirror-generator</a></p> <p>A suggested version file can be found in <a href="/suggest_versions.txt">suggest_versions.txt.</a></p> - """ - ) + """) + for resource in resource_list: + print("Processing " + str(resource)) if "file_name" in resource.keys(): file_name = resource['file_name'] else: diff --git a/utils/github.py b/utils/github.py index 788edf6..7c0cfd7 100644 --- a/utils/github.py +++ b/utils/github.py @@ -41,6 +41,7 @@ def download_repo_by_tag(owner_name: str, repo_name: str, archive_type: str = "t "file_name": f"{repo_name}-{tag}.{archive_type}", "version": tag }) + resource_list.reverse() if latest_meta_name: latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]} else: @@ -116,7 +117,7 @@ def get_package_from_release_with_regular_expression(owner_name: str, repo_name: if len(resource_list) == 0: raise ValueError("No asset matches regex") - if not latest_meta_name: + if latest_meta_name: latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]} else: latest_meta = None diff --git a/utils/imagemagick.py b/utils/imagemagick.py index 62a2198..2807063 100644 --- a/utils/imagemagick.py +++ b/utils/imagemagick.py @@ -1,11 +1,15 @@ +from typing import Tuple, List, Dict, Any + import httpx from bs4 import BeautifulSoup -def make_cache() -> list: +def make_cache() -> tuple[list[dict[str, str | Any]], dict[str, str | Any]]: url = "https://imagemagick.org/archive/" soup = BeautifulSoup(httpx.get(url).text, "html.parser") resource_list = [] + latest_meta = {"version_file_name": "imagemagick_ver"} + for a in soup.find_all("a", href=True): if a["href"].startswith("ImageMagick-") and a["href"].endswith(".tar.gz"): resource_list.append({ @@ -14,4 +18,5 @@ def make_cache() -> list: "gpg": url + a["href"] + ".asc", "file_name": a["href"].split("/")[-1] }) - return resource_list + latest_meta["version"] = resource_list[0]["version"] + return resource_list, latest_meta diff --git a/utils/php_plugins.py b/utils/php_plugins.py index 2356c58..b2bad43 100644 --- a/utils/php_plugins.py +++ b/utils/php_plugins.py @@ -1,14 +1,16 @@ import time +from typing import Tuple, List, Dict, Any import httpx from bs4 import BeautifulSoup MAX_TRIES = 50 -ALLOWED_NUMBER_OF_VERSIONS = 3 BLACKLIST_WORD = ["alpha", "beta", "rc", "test"] -def make_cache(package_name: str, file_prefix: str) -> list: +def make_cache(package_name: str, file_prefix: str, allow_unstable_version: bool = False, + latest_meta_name: str = None) \ + -> tuple[list[dict[str, str | Any]], dict[str, str] | None] | list[Any]: tried = 0 while tried < MAX_TRIES: try: @@ -17,14 +19,19 @@ def make_cache(package_name: str, file_prefix: str) -> list: soup = BeautifulSoup(httpx.get(url).text, "html.parser") for a in soup.find_all("a"): if a.text.startswith(f"{file_prefix}-") and a.text.endswith(".tgz"): - if any(word in a.text for word in BLACKLIST_WORD): - continue + if not allow_unstable_version: + if any(word in a.text.lower() for word in BLACKLIST_WORD): + continue resource_list.append({ "version": a.text.replace(f"{file_prefix}-", "").replace(".tgz", ""), "url": f"https://pecl.php.net" + a["href"] if a["href"].startswith("/") else a["href"], "file_name": a.text }) - return resource_list[:ALLOWED_NUMBER_OF_VERSIONS] + if latest_meta_name: + latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]} + else: + latest_meta = None + return resource_list, latest_meta except httpx.ReadTimeout: tried += 1 print(f"Retrying to download PHP plugin {package_name}: {tried}/{MAX_TRIES}")