From 3a15707be838477116f8d266d2d85b55b1728998 Mon Sep 17 00:00:00 2001 From: Andreas Eknes Lie Date: Thu, 16 Jan 2025 15:06:51 +0100 Subject: [PATCH] Update transpiling to also cover an optional extra argument --- komodo/release_transpiler.py | 112 ++++++++++++++++++++++++------- tests/test_release_transpiler.py | 13 +++- 2 files changed, 98 insertions(+), 27 deletions(-) diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py index 6ace9e66..76b85bda 100755 --- a/komodo/release_transpiler.py +++ b/komodo/release_transpiler.py @@ -2,9 +2,11 @@ import argparse import os -from typing import Dict, Union +import re +from typing import Dict, Union, Optional import yaml +from typing_extensions import Sequence from komodo.matrix import format_release, get_matrix from komodo.prettier import load_yaml, write_to_file @@ -14,6 +16,7 @@ def _pick_package_versions_for_release( packages: dict, rhel_ver: str, py_ver: str, + other_ver: Optional[str] = None, ) -> dict: """Consolidate the packages for a given combination of rhel and python version into a dictionary. @@ -22,7 +25,7 @@ def _pick_package_versions_for_release( for pkg_name, versions in packages.items(): version = None try: - _check_version_exists_for_coordinates(versions, rhel_ver, py_ver) + _check_version_exists_for_coordinates(versions, rhel_ver, py_ver, other_ver) except KeyError as err: error_msg = f"{err!s}. Failed for {pkg_name}." raise KeyError(error_msg) from None @@ -31,6 +34,12 @@ def _pick_package_versions_for_release( version = versions[rhel_ver][py_ver] elif py_ver in versions: version = versions[py_ver] + + if other_ver: + if other_ver in versions: + version = versions[other_ver] + elif other_ver and other_ver in version: + version = version[other_ver] else: version = versions if version: @@ -42,6 +51,7 @@ def _check_version_exists_for_coordinates( pkg_versions: Union[dict, str], rhel_coordinate: str, py_coordinate: str, + other_coordinate: Optional[str], ) -> None: """Check the coordinates `rhel_ver` and `py_ver` input as arguments to build a release against the release matrix file. Raise exceptions if @@ -62,27 +72,58 @@ def _check_version_exists_for_coordinates( } or: {1.1.1}. - + or: + { + py36: 1.1.1, # first level + numpy1: 1.2.6 + numpy2: 2.2.1 + py38: 2.1.1, + numpy1: 1.2.6 + numpy2: 2.2.1 + } + or: + { + numpy2: 2.2.1 + numpy1: 1.2.6 + } """ if isinstance(pkg_versions, str): return None first_level_versions = list(pkg_versions) + + def verify_coordinate_in_list( + coordinate: str, all_coordinates: Sequence[str], seq: Sequence[str] + ) -> None: + if not coordinate in seq: + raise KeyError( + f"Matrix coordinate {coordinate}, part of {all_coordinates}, not found in {seq}" + ) + + all_coords = [rhel_coordinate, py_coordinate, other_coordinate] + if "rhel" in first_level_versions[0]: - # Both rhel and python versions can have different versions - if rhel_coordinate not in first_level_versions: - msg = f"Rhel version {rhel_coordinate} not found." - raise KeyError(msg) + verify_coordinate_in_list(rhel_coordinate, all_coords, first_level_versions) second_level_versions = list(pkg_versions[rhel_coordinate]) - if py_coordinate not in second_level_versions: - msg = f"Python version {py_coordinate} not found for rhel version {rhel_coordinate}." - raise KeyError( - msg, + verify_coordinate_in_list(py_coordinate, all_coords, second_level_versions) + + if other_coordinate: + third_level_versions = list(pkg_versions[rhel_coordinate][py_coordinate]) + verify_coordinate_in_list( + other_coordinate, all_coords, third_level_versions + ) + + elif re.match(r"py\d{2,3}", first_level_versions[0]): + verify_coordinate_in_list(py_coordinate, all_coords, first_level_versions) + + if other_coordinate: + second_level_versions = list(pkg_versions[py_coordinate]) + verify_coordinate_in_list( + other_coordinate, all_coords, second_level_versions ) - elif "py" in first_level_versions[0]: - if py_coordinate not in first_level_versions: - # Only python has different versions - msg = f"Python version {py_coordinate} not found." - raise KeyError(msg) + + elif other_coordinate: + verify_coordinate_in_list(other_coordinate, all_coords, first_level_versions) + else: msg = """Invalid package versioning structure.""" raise KeyError(msg) @@ -95,19 +136,32 @@ def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> No Write one dimension file for each element in the matrix (e.g. rhel7 and py3.8, rhel6 and py3.6). """ - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] + rhel_versions = matrix.get("rhel", "8") + python_versions = matrix.get("py", "311") + other_versions = None + + for k, v in matrix.items(): # find first item not rhel or py + if k not in ["rhel", "py"]: + other_versions = {k: v} + break release_base = os.path.splitext(os.path.basename(matrix_file))[0] release_folder = os.path.dirname(matrix_file) release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + + for rhel_ver, py_ver, other_ver in get_matrix( + rhel_versions, python_versions, other_versions + ): release_dict = _pick_package_versions_for_release( release_matrix, rhel_ver, py_ver, + other_ver, ) - filename = f"{format_release(release_base, rhel_ver, py_ver)}.yml" + filename = f"{format_release(release_base, rhel_ver, py_ver)}" + if other_versions: + filename = filename + f"-{other_ver}" + filename = filename + ".yml" write_to_file(release_dict, os.path.join(output_folder, filename)) @@ -117,24 +171,34 @@ def transpile_releases_for_pip( repository_file: str, matrix: dict, ) -> None: - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] + rhel_versions = matrix.get("rhel", "8") + python_versions = matrix.get("py", "311") + other_versions = None + + for k, v in matrix.items(): # find first item not rhel or py + if k not in ["rhel", "py"]: + other_versions = {k: v} + break + release_base = os.path.splitext(os.path.basename(matrix_file))[0] release_folder = os.path.dirname(matrix_file) release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") repository = load_yaml(repository_file) - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + for rhel_ver, py_ver, other_ver in get_matrix( + rhel_versions, python_versions, other_versions + ): release_dict = _pick_package_versions_for_release( release_matrix, rhel_ver, py_ver, + other_ver, ) pip_packages = [ f"{pkg}=={version}" for pkg, version in release_dict.items() if repository[pkg][version].get("make") == "pip" ] - filename = f"{format_release(release_base, rhel_ver, py_ver)}.req" + filename = f"{format_release(release_base, rhel_ver, py_ver, other_ver)}.req" with open( os.path.join(output_folder, filename), mode="w", diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py index fbe05f66..827742d0 100644 --- a/tests/test_release_transpiler.py +++ b/tests/test_release_transpiler.py @@ -22,7 +22,10 @@ @pytest.mark.parametrize( "matrix", - [({"py": ["3.8"], "rhel": ["7"]}), ({"py": ["3.8", "3.11"], "rhel": ["7", "8"]})], + [ + ({"py": ["3.8"], "rhel": ["7"]}), + ({"py": ["3.8", "3.11"], "rhel": ["7", "8"]}), + ], ) def test_transpile_add_argument(tmpdir, matrix): release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") @@ -50,8 +53,12 @@ def test_transpile_add_argument(tmpdir, matrix): {"py": ["3.6"], "rhel": ["5"]}, ["rhel5", "lib1"], ), + ( + {"py": ["3.6"], "rhel": ["7"], "numpy": ["2"]}, + ["numpy2", "lib1"], + ), ], - ids=["Pass for all packages", "Fail", "Fail"], + ids=["Pass for all packages", "Fail", "Fail", "Missing custom numpy coordinate"], ) def test_check_version_exists_for_coordinates(matrix, error_message_content, tmpdir): release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") @@ -59,7 +66,7 @@ def test_check_version_exists_for_coordinates(matrix, error_message_content, tmp with tmpdir.as_cwd(): transpile_releases(release_file, os.getcwd(), matrix) except KeyError as exception_info: - assert all(word in str(exception_info) for word in error_message_content) + assert all(word in str(exception_info.args) for word in error_message_content) def test_transpile_for_pip(tmpdir):