Skip to content

Commit

Permalink
add version.txt support for GitHub sources
Browse files Browse the repository at this point in the history 
#3
  • Loading branch information
@Masterain98
Masterain98 committed Oct 22, 2023
1 parent 39a99db commit 89bfcba
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 36 deletions.
62 changes: 36 additions & 26 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,41 +79,53 @@ def main():
resource_list += openssl_output[0]
latest_meta_list += openssl_output[1]

resource_list += github.download_repo_by_tag("openresty", "lua-resty-core",
"tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "lua-resty-core", "tar.gz", True)
resource_list += pure_ftpd.make_cache()
resource_list += htop.make_cache()
resource_list += github.get_single_package_from_release("jemalloc", "jemalloc")
resource_list += github.download_repo_by_tag("openresty", "lua-resty-lrucache",
"tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "luajit2",
"tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "lua-cjson",
"tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "lua-resty-lrucache", "tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "luajit2", "tar.gz", True)
resource_list += github.download_repo_by_tag("openresty", "lua-cjson", "tar.gz", True)
resource_list += github.get_package_from_release_with_regular_expression("gperftools",
"gperftools",
r"gperftools-\d+.\d+.tar.gz",
3)
resource_list += github.get_package_from_release_with_regular_expression("unicode-org",
"icu",
r"(icu4c-)[\d|\-|\_]+(src\.tgz)",
3)

icu_output = github.get_package_from_release_with_regular_expression("unicode-org",
"icu",
r"(icu4c-)[\d|\-|\_]+(src\.tgz)",
3,
"icu4c")
resource_list += icu_output[0]
latest_meta_list.append(icu_output[1])

# gdrive package is changed!!!
resource_list += github.get_package_from_release_with_regular_expression("glotlabs",
"gdrive",
r"linux",
1)
resource_list += github.get_package_from_release_with_regular_expression("nih-at",
"libzip",
r"\.tar\.gz",
5)
resource_list += github.get_package_from_release_with_regular_expression("jedisct1",
"libsodium",
r"\d+\.tar\.gz",
5)
libzip_output = github.get_package_from_release_with_regular_expression("nih-at",
"libzip",
r"\.tar\.gz",
5,
"libzip_ver")
resource_list += libzip_output[0]
latest_meta_list.append(libzip_output[1])

libsodium_output = github.get_package_from_release_with_regular_expression("jedisct1",
"libsodium",
r"\d+\.tar\.gz",
5,
"libsodium_ver")
resource_list += libsodium_output[0]
latest_meta_list.append(libsodium_output[1])

# Name changed!!! Was argon2-20190702.tar.gz and 20190702.tar.gz
resource_list += github.download_repo_by_tag("P-H-C", "phc-winner-argon2",
archive_type="tar.gz", filter_blacklist=True)
argon2_output = github.download_repo_by_tag("P-H-C", "phc-winner-argon2",
archive_type="tar.gz", filter_blacklist=True,
latest_meta_name="argon2_ver")
resource_list += argon2_output[0]
latest_meta_list.append(argon2_output[1])

freetype_output = freetype.make_cache()
resource_list += freetype_output[0]
Expand All @@ -123,11 +135,9 @@ def main():
"libevent",
r"\.tar\.gz$",
5)
resource_list += github.download_repo_by_tag("jokkedk", "webgrind",
"zip", False)
resource_list += github.download_repo_by_tag("jokkedk", "webgrind", "zip", False)
# ngx_devel_kit name changed!!!
resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit",
"tar.gz", False)
resource_list += github.download_repo_by_tag("vision5", "ngx_devel_kit", "tar.gz", False)
resource_list += github.get_package_from_release_with_regular_expression("kkos", "oniguruma",
r"\.tar\.gz$", 5)
resource_list += github.get_package_from_release_with_regular_expression("dropbox", "dbxcli",
Expand Down
44 changes: 34 additions & 10 deletions utils/github.py
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from typing import Tuple, List, Dict, Any

import httpx
import re

BLACKLIST_WORD = ["rc", "beta", "alpha"]


def download_repo_by_tag(owner_name: str, repo_name: str, archive_type: str = "tar.gz",
filter_blacklist: bool = True) -> list:
filter_blacklist: bool = True, latest_meta_name: str = None) -> tuple[list[dict[str, str | Any]], dict[str, str | Any]]:
"""
Download repository archive by tag
Expand All @@ -14,6 +16,7 @@ def download_repo_by_tag(owner_name: str, repo_name: str, archive_type: str = "t
This function is suitable for GitHub repositories that does not make any releases and package is
repository content itself
:param latest_meta_name: Package name for latest meta returning
:param owner_name: GitHub account name
:param repo_name: repository name, e.g. "alibaba/tengine"
:param archive_type: "tar.gz" or "zip"
Expand All @@ -35,19 +38,25 @@ def download_repo_by_tag(owner_name: str, repo_name: str, archive_type: str = "t
tag_archive_url = f"https://github.com/{owner_name}/{repo_name}/archive/refs/tags/{tag}.{archive_type}"
resource_list.append({
"url": tag_archive_url,
"file_name": f"{repo_name}-{tag}.{archive_type}"
"file_name": f"{repo_name}-{tag}.{archive_type}",
"version": tag
})
return resource_list
if latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
latest_meta = None
return resource_list, latest_meta


def get_single_package_from_release(owner_name: str, repo_name: str):
def get_single_package_from_release(owner_name: str, repo_name: str, latest_meta_name: str = None) -> tuple[list[dict[str, str | Any]], dict[str, str | None | Any] | None]:
"""
Get single package from GitHub release
This function will get release and download the package
This function is suitable for GitHub repositories that make releases and only one file in each release
:param latest_meta_name: Package name for latest meta returning
:param owner_name: GitHub account name
:param repo_name: repository name, e.g. "alibaba/tengine"
:return: list of dict, each dict contains at least "url" and "file_name"
Expand All @@ -61,21 +70,28 @@ def get_single_package_from_release(owner_name: str, repo_name: str):
continue
resource_list.append({
"url": release["assets"][0]["browser_download_url"],
"file_name": release["assets"][0]["name"]
"file_name": release["assets"][0]["name"],
"version": release["tag_name"]
})
else:
raise ValueError("More than one file in release")
return resource_list
if latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
latest_meta = None
return resource_list, latest_meta


def get_package_from_release_with_regular_expression(owner_name: str, repo_name: str, regex: str, max_asset: int = 0):
def get_package_from_release_with_regular_expression(owner_name: str, repo_name: str, regex: str, max_asset: int = 0,
latest_meta_name: str = None) -> tuple[list[dict[str, Any]], dict[str, str | None | Any] | None]:
"""
Get single package from GitHub release with regular expression
This function will get release and download the package
This function is suitable for GitHub repositories that make releases and only one file in each release
:param latest_meta_name: Package name for latest meta returning
:param owner_name: GitHub account name
:param repo_name: repository name, e.g. "alibaba/tengine"
:param regex: regular expression to match file name
Expand All @@ -84,6 +100,7 @@ def get_package_from_release_with_regular_expression(owner_name: str, repo_name:
"""
if regex is None:
raise ValueError("regex must be specified")

resource_list = []
url = f"https://api.github.com/repos/{owner_name}/{repo_name}/releases"
releases = httpx.get(url).json()
Expand All @@ -93,11 +110,18 @@ def get_package_from_release_with_regular_expression(owner_name: str, repo_name:
if re.search(regex, asset["name"]):
resource_list.append({
"url": asset["browser_download_url"],
"file_name": asset["name"]
"file_name": asset["name"],
"version": release["tag_name"]
})
if len(resource_list) == 0:
raise ValueError("No asset matches regex")

if not latest_meta_name:
latest_meta = {"version_file_name": latest_meta_name, "version": resource_list[0]["version"]}
else:
latest_meta = None

if max_asset > 0:
return resource_list[:max_asset]
return resource_list[:max_asset], latest_meta
else:
return resource_list
return resource_list, latest_meta

0 comments on commit 89bfcba

Please sign in to comment.