Skip to content

Commit

Permalink
Update scripts
Browse files Browse the repository at this point in the history 
#3
  • Loading branch information
@Masterain98
Masterain98 committed Oct 23, 2023
1 parent 89bfcba commit dee8c22
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 34 deletions.
122 changes: 96 additions & 26 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ def main():
resource_list += apr_output[0]
latest_meta_list += apr_output[1]

resource_list += imagemagick.make_cache()
imagemagick_output = imagemagick.make_cache()
resource_list += imagemagick_output[0]
latest_meta_list.append(imagemagick_output[1])

openresty_output = openresty.make_cache()
resource_list += openresty_output[0]
Expand All @@ -79,31 +81,51 @@ def main():
resource_list += openssl_output[0]
latest_meta_list += openssl_output[1]

resource_list += github.download_repo_by_tag("openresty", "lua-resty-core", "tar.gz", True)
lua_resty_core_output = github.download_repo_by_tag("openresty", "lua-resty-core",
"tar.gz", True, "lua_resty_core_ver")
resource_list += lua_resty_core_output[0]
latest_meta_list.append(lua_resty_core_output[1])

resource_list += pure_ftpd.make_cache()
resource_list += htop.make_cache()
resource_list += github.get_single_package_from_release("jemalloc", "jemalloc")
resource_list += github.download_repo_by_tag("openresty", "lua-resty-lrucache", "tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "luajit2", "tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "lua-cjson", "tar.gz", True)

jemalloc_output = github.get_single_package_from_release("jemalloc", "jemalloc", "jemalloc_ver")
resource_list += jemalloc_output[0]
latest_meta_list.append(jemalloc_output[1])

lua_resty_lrucache_output = github.download_repo_by_tag("openresty", "lua-resty-lrucache", "tar.gz", True,
"lua_resty_lrucache_ver")
resource_list += lua_resty_lrucache_output[0]
latest_meta_list.append(lua_resty_lrucache_output[1])

luajit2_output = github.download_repo_by_tag("openresty", "luajit2", "tar.gz", True, "luajit2_ver")
resource_list += luajit2_output[0]
latest_meta_list.append(luajit2_output[1])

lua_cjson_output = github.download_repo_by_tag("openresty", "lua-cjson", "tar.gz", True, "lua_cjson_ver")
resource_list += lua_cjson_output[0]
latest_meta_list.append(lua_cjson_output[1])

resource_list += github.get_package_from_release_with_regular_expression("gperftools",
"gperftools",
r"gperftools-\d+.\d+.tar.gz",
3)
3,
None)[0]

icu_output = github.get_package_from_release_with_regular_expression("unicode-org",
"icu",
r"(icu4c-)[\d|\-|\_]+(src\.tgz)",
3,
"icu4c")
"icu4c_ver")
resource_list += icu_output[0]
latest_meta_list.append(icu_output[1])

# gdrive package is changed!!!
resource_list += github.get_package_from_release_with_regular_expression("glotlabs",
"gdrive",
r"linux",
1)
1,
None)[0]
libzip_output = github.get_package_from_release_with_regular_expression("nih-at",
"libzip",
r"\.tar\.gz",
Expand Down Expand Up @@ -134,16 +156,17 @@ def main():
resource_list += github.get_package_from_release_with_regular_expression("libevent",
"libevent",
r"\.tar\.gz$",
5)
resource_list += github.download_repo_by_tag("jokkedk", "webgrind", "zip", False)
5,
None)[0]
resource_list += github.download_repo_by_tag("jokkedk", "webgrind", "zip", False, None)[0]
# ngx_devel_kit name changed!!!
resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit", "tar.gz", False)
resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit", "tar.gz", False, None)[0]
resource_list += github.get_package_from_release_with_regular_expression("kkos", "oniguruma",
r"\.tar\.gz$", 5)
r"\.tar\.gz$", 5, None)[0]
resource_list += github.get_package_from_release_with_regular_expression("dropbox", "dbxcli",
r"dbxcli-linux-arm", 1)
r"dbxcli-linux-arm", 1, None)[0]
resource_list += github.get_package_from_release_with_regular_expression("dropbox", "dbxcli",
r"dbxcli-linux-amd64", 1)
r"dbxcli-linux-amd64", 1, None)[0]
resource_list += bison.make_cache()

libiconv_output = libiconv.make_cache()
Expand All @@ -154,15 +177,61 @@ def main():
resource_list += misc_output[0]
latest_meta_list += misc_output[1]

resource_list += php_plugins.make_cache("APCU", "apcu")
resource_list += php_plugins.make_cache("gmagick", "gmagick")
resource_list += php_plugins.make_cache("imagick", "imagick")
resource_list += php_plugins.make_cache("memcache", "memcache")
resource_list += php_plugins.make_cache("mongodb", "mongodb")
resource_list += php_plugins.make_cache("swoole", "swoole")
resource_list += php_plugins.make_cache("YAF", "yaf")
resource_list += php_plugins.make_cache("xdebug", "xdebug")
resource_list += php_plugins.make_cache("mongo", "mongo")
apcu_output = php_plugins.make_cache("APCU", "apcu",
False, "apcu_ver")
resource_list += apcu_output[0]
latest_meta_list.append(apcu_output[1])

gmagick_output = php_plugins.make_cache("gmagick", "gmagick",
True, "gmagick_ver")
resource_list += gmagick_output[0]
latest_meta_list.append(gmagick_output[1])

imagick_output = php_plugins.make_cache("imagick", "imagick",
False, "imagick_ver")
resource_list += imagick_output[0]
latest_meta_list.append(imagick_output[1])

pecl_memcache_output = php_plugins.make_cache("memcache", "memcache",
False, "pecl_memcache_ver")
resource_list += pecl_memcache_output[0]
latest_meta_list.append(pecl_memcache_output[1])

pecl_mongodb_output = php_plugins.make_cache("mongodb", "mongodb",
False, "pecl_mongodb_ver")
resource_list += pecl_mongodb_output[0]
latest_meta_list.append(pecl_mongodb_output[1])

swoole_output = php_plugins.make_cache("swoole", "swoole",
False, "swoole_ver")
resource_list += swoole_output[0]
latest_meta_list.append(swoole_output[1])

yaf_output = php_plugins.make_cache("YAF", "yaf",
False, "yaf_ver")
resource_list += yaf_output[0]
latest_meta_list.append(yaf_output[1])

xdebug_output = php_plugins.make_cache("xdebug", "xdebug",
False, "xdebug_ver")
resource_list += xdebug_output[0]
latest_meta_list.append(xdebug_output[1])

pecl_mongo_output = php_plugins.make_cache("mongo", "mongo",
False, "pecl_mongo_ver")
resource_list += pecl_mongo_output[0]
latest_meta_list.append(pecl_mongo_output[1])

# Older versions of PHP plugins
latest_meta_list += [
{"version_file_name": "apcu_oldver", "version": "4.0.11"},
{"version_file_name": "gmagick_oldver", "version": "1.1.7RC3"},
{"version_file_name": "imagick_oldver", "version": "3.4.4"},
{"version_file_name": "pecl_memcache_oldver", "version": "4.0.5.2"},
{"version_file_name": "pecl_mongodb_oldver", "version": "1.9.2"},
{"version_file_name": "swoole_oldver", "version": "4.8.12"},
{"version_file_name": "xdebug_oldver", "version": "2.9.8"},
]

with open(r"./output/resources.json", "w+") as f:
f.write(json.dumps(resource_list, indent=4))
Expand All @@ -187,9 +256,10 @@ def main():
<h1>Oneinstack Mirror</h1>
<p>This page is generated by <a href="https://github.com/dalao-org/oneinstack-mirror-generator">oneinstack-mirror-generator</a></p>
<p>A suggested version file can be found in <a href="/suggest_versions.txt">suggest_versions.txt.</a></p>
"""
)
""")

for resource in resource_list:
print("Processing " + str(resource))
if "file_name" in resource.keys():
file_name = resource['file_name']
else:
Expand Down
3 changes: 2 additions & 1 deletion utils/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ def download_repo_by_tag(owner_name: str, repo_name: str, archive_type: str = "t
"file_name": f"{repo_name}-{tag}.{archive_type}",
"version": tag
})
resource_list.reverse()
if latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
Expand Down Expand Up @@ -116,7 +117,7 @@ def get_package_from_release_with_regular_expression(owner_name: str, repo_name:
if len(resource_list) == 0:
raise ValueError("No asset matches regex")

if not latest_meta_name:
if latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
latest_meta = None
Expand Down
9 changes: 7 additions & 2 deletions utils/imagemagick.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
from typing import Tuple, List, Dict, Any

import httpx
from bs4 import BeautifulSoup


def make_cache() -> list:
def make_cache() -> tuple[list[dict[str, str | Any]], dict[str, str | Any]]:
url = "https://imagemagick.org/archive/"
soup = BeautifulSoup(httpx.get(url).text, "html.parser")
resource_list = []
latest_meta = {"version_file_name": "imagemagick_ver"}

for a in soup.find_all("a", href=True):
if a["href"].startswith("ImageMagick-") and a["href"].endswith(".tar.gz"):
resource_list.append({
Expand All @@ -14,4 +18,5 @@ def make_cache() -> list:
"gpg": url + a["href"] + ".asc",
"file_name": a["href"].split("/")[-1]
})
return resource_list
latest_meta["version"] = resource_list[0]["version"]
return resource_list, latest_meta
17 changes: 12 additions & 5 deletions utils/php_plugins.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import time
from typing import Tuple, List, Dict, Any

import httpx
from bs4 import BeautifulSoup

MAX_TRIES = 50
ALLOWED_NUMBER_OF_VERSIONS = 3
BLACKLIST_WORD = ["alpha", "beta", "rc", "test"]


def make_cache(package_name: str, file_prefix: str) -> list:
def make_cache(package_name: str, file_prefix: str, allow_unstable_version: bool = False,
latest_meta_name: str = None) \
-> tuple[list[dict[str, str | Any]], dict[str, str] | None] | list[Any]:
tried = 0
while tried < MAX_TRIES:
try:
Expand All @@ -17,14 +19,19 @@ def make_cache(package_name: str, file_prefix: str) -> list:
soup = BeautifulSoup(httpx.get(url).text, "html.parser")
for a in soup.find_all("a"):
if a.text.startswith(f"{file_prefix}-") and a.text.endswith(".tgz"):
if any(word in a.text for word in BLACKLIST_WORD):
continue
if not allow_unstable_version:
if any(word in a.text.lower() for word in BLACKLIST_WORD):
continue
resource_list.append({
"version": a.text.replace(f"{file_prefix}-", "").replace(".tgz", ""),
"url": f"https://pecl.php.net" + a["href"] if a["href"].startswith("/") else a["href"],
"file_name": a.text
})
return resource_list[:ALLOWED_NUMBER_OF_VERSIONS]
if latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
latest_meta = None
return resource_list, latest_meta
except httpx.ReadTimeout:
tried += 1
print(f"Retrying to download PHP plugin {package_name}: {tried}/{MAX_TRIES}")
Expand Down

0 comments on commit dee8c22

Please sign in to comment.