diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 70835d175..b06c1b683 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ jobs: build: env: RELEASE_NAME: release-matrix + RELEASE_FULL_NAME: release-matrix.yml REPOSITORY: repository.yml strategy: @@ -42,19 +43,19 @@ jobs: - name: Lint examples run: | - komodo-lint examples/stable.yml examples/repository.yml + komodo-lint examples/stable.yml examples/${{env.REPOSITORY}} - name: Transpile file run: | komodo-transpiler transpile \ - --matrix-file="ci/${{env.RELEASE_NAME}}.yml" \ + --matrix-file=ci/"${{ env.RELEASE_FULL_NAME }}" \ --output ci --matrix-coordinates "{rhel: ['7'], py: ['${{matrix.python-version}}']}" - + - name: Full integration test run: | py_version_number=$(echo "${{ matrix.python-version }}" | sed 's/\.//g') kmd \ - ci/${{env.RELEASE_NAME}}-py$py_version_number-rhel7.yml \ + ci/${{ env.RELEASE_NAME }}-py$py_version_number-rhel7.yml \ ci/${{env.REPOSITORY}} \ --workspace ${{ runner.temp }}/kmd-ws \ --prefix ${{ runner.temp }}/prefix \ diff --git a/komodo/check_up_to_date_pypi.py b/komodo/check_up_to_date_pypi.py index db22d7660..6ca5de0ef 100644 --- a/komodo/check_up_to_date_pypi.py +++ b/komodo/check_up_to_date_pypi.py @@ -182,9 +182,7 @@ def run_check_up_to_date( release_file, repository_file, python_version=( - f"{sys.version_info.major}." - f"{sys.version_info.minor}." - f"{sys.version_info.micro}" + f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" ), propose_upgrade=False, ignore=None, diff --git a/komodo/cli.py b/komodo/cli.py index e47e836bf..1ed34c3f3 100755 --- a/komodo/cli.py +++ b/komodo/cli.py @@ -232,7 +232,7 @@ def rsync_komodo_to_destination(release_name: str, destination: str) -> None: def move_old_release_from_release_path_if_exists(release_path: Path) -> None: if release_path.exists(): - shell(f"mv {str(release_path)} " f"{str(release_path)}.delete-{uuid.uuid4()}") + shell(f"mv {str(release_path)} {str(release_path)}.delete-{uuid.uuid4()}") def move_new_release_to_release_path(args: KomodoNamespace, release_path: Path) -> None: diff --git a/komodo/matrix.py b/komodo/matrix.py index a1c3bee9f..804949fa9 100644 --- a/komodo/matrix.py +++ b/komodo/matrix.py @@ -6,35 +6,53 @@ import itertools import re -from typing import Iterator, Sequence, Tuple +from typing import Dict, Iterator, Optional, Sequence, Tuple def get_matrix( rhel_versions: Sequence[str], py_versions: Sequence[str], -) -> Iterator[Tuple[str, str]]: - """Return tuples of rhel version and Python version, representing the - current release matrix. + custom_coordinate: Optional[Dict[str, Sequence[str]]] = None, +) -> Iterator[Tuple[str, str, str]]: + """Return tuples of rhel version, Python version and a single optional custom_coordinate, + representing the current release matrix. """ - for product in itertools.product(rhel_versions, py_versions): - rh_ver, py_ver = product - yield (f"rhel{rh_ver}", f"py{str(py_ver).replace('.', '')}") + component_name = "" + component_seq = [None] + if custom_coordinate: + if len(custom_coordinate) != 1: + raise ValueError("custom_coordinate must contain exactly one item") + component_name, component_seq = next(iter(custom_coordinate.items())) # pylint: disable=stop-iteration-return -def format_release(base: str, rhel_ver: str, py_ver: str) -> str: + for product in itertools.product(rhel_versions, py_versions, component_seq): + rh_ver, py_ver, other_ver = product + rhel_tag, py_tag, other_tag = ( + f"rhel{rh_ver}", + f"py{str(py_ver).replace('.', '')}", + f"{component_name}{other_ver}" if other_ver else None, + ) + + yield rhel_tag, py_tag, other_tag + + +def format_release( + base: str, rhel_ver: str, py_ver: str, other_component: Optional[str] = None +) -> str: """Format a base (e.g. a matrix file without the .yml suffix) such that it looks like a concrete release. """ - return f"{base}-{py_ver}-{rhel_ver}" + return ( + f"{base}-{py_ver}-{rhel_ver}-{other_component}" + if other_component + else f"{base}-{py_ver}-{rhel_ver}" + ) def get_matrix_base(release_name: str) -> str: - """Return the base (e.g. matrix part of a concrete release). Should be the - inverse of format_release for actual, concrete matrix releases. - Hard-coded the suffix pattern '-py..-rhel.' or '-py...-rhel.'. + """Return the base (e.g. matrix part of a concrete release). + Match release name on -py[nno]-rhel[n] and delimit using that """ - suffix = format_release("", "rhel[0-9]", r"py\d{2,3}") - if re.search(suffix, release_name): - return re.split(suffix, release_name)[0] - # no matrix suffix at all + if re.search(r"-py\d{2,3}-rhel[0-9]", release_name): + return release_name.split("-py")[0] return release_name diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py index dcfd8affa..00bb2e9db 100755 --- a/komodo/release_transpiler.py +++ b/komodo/release_transpiler.py @@ -2,7 +2,8 @@ import argparse import os -from typing import Dict, Sequence, Union +import re +from typing import Dict, Optional, Sequence, Union import yaml @@ -10,27 +11,11 @@ from komodo.prettier import load_yaml, write_to_file -def get_py_coords(release_base: str, release_folder: str) -> Sequence[str]: - """Get python versions of release files inside a given release_folder.""" - filenames_with_prefix = sorted( - [ - filename - for filename in os.listdir(release_folder) - if filename.startswith(release_base) - ], - ) - len_release_base = len(release_base + "-") - irrelevant_suffix_length = len(".yml") - return [ - filename[len_release_base:-irrelevant_suffix_length] - for filename in filenames_with_prefix - ] - - def _pick_package_versions_for_release( packages: dict, rhel_ver: str, py_ver: str, + other_ver: Optional[str] = None, ) -> dict: """Consolidate the packages for a given combination of rhel and python version into a dictionary. @@ -39,7 +24,7 @@ def _pick_package_versions_for_release( for pkg_name, versions in packages.items(): version = None try: - _check_version_exists_for_coordinates(versions, rhel_ver, py_ver) + _check_version_exists_for_coordinates(versions, rhel_ver, py_ver, other_ver) except KeyError as err: error_msg = f"{err!s}. Failed for {pkg_name}." raise KeyError(error_msg) from None @@ -48,6 +33,12 @@ def _pick_package_versions_for_release( version = versions[rhel_ver][py_ver] elif py_ver in versions: version = versions[py_ver] + + if other_ver: + if other_ver in versions: + version = versions[other_ver] + elif other_ver and other_ver in version: + version = version[other_ver] else: version = versions if version: @@ -59,6 +50,7 @@ def _check_version_exists_for_coordinates( pkg_versions: Union[dict, str], rhel_coordinate: str, py_coordinate: str, + other_coordinate: Optional[str], ) -> None: """Check the coordinates `rhel_ver` and `py_ver` input as arguments to build a release against the release matrix file. Raise exceptions if @@ -79,27 +71,58 @@ def _check_version_exists_for_coordinates( } or: {1.1.1}. - + or: + { + py36: 1.1.1, # first level + numpy1: 1.2.6 + numpy2: 2.2.1 + py38: 2.1.1, + numpy1: 1.2.6 + numpy2: 2.2.1 + } + or: + { + numpy2: 2.2.1 + numpy1: 1.2.6 + } """ if isinstance(pkg_versions, str): return None first_level_versions = list(pkg_versions) + + def verify_coordinate_in_list( + coordinate: str, all_coordinates: Sequence[str], seq: Sequence[str] + ) -> None: + if not coordinate in seq: + raise KeyError( + f"Matrix coordinate {coordinate}, part of {all_coordinates}, not found in {seq}" + ) + + all_coords = [rhel_coordinate, py_coordinate, other_coordinate] + if "rhel" in first_level_versions[0]: - # Both rhel and python versions can have different versions - if rhel_coordinate not in first_level_versions: - msg = f"Rhel version {rhel_coordinate} not found." - raise KeyError(msg) + verify_coordinate_in_list(rhel_coordinate, all_coords, first_level_versions) second_level_versions = list(pkg_versions[rhel_coordinate]) - if py_coordinate not in second_level_versions: - msg = f"Python version {py_coordinate} not found for rhel version {rhel_coordinate}." - raise KeyError( - msg, + verify_coordinate_in_list(py_coordinate, all_coords, second_level_versions) + + if other_coordinate: + third_level_versions = list(pkg_versions[rhel_coordinate][py_coordinate]) + verify_coordinate_in_list( + other_coordinate, all_coords, third_level_versions ) - elif "py" in first_level_versions[0]: - if py_coordinate not in first_level_versions: - # Only python has different versions - msg = f"Python version {py_coordinate} not found." - raise KeyError(msg) + + elif re.match(r"py\d{2,3}", first_level_versions[0]): + verify_coordinate_in_list(py_coordinate, all_coords, first_level_versions) + + if other_coordinate: + second_level_versions = list(pkg_versions[py_coordinate]) + verify_coordinate_in_list( + other_coordinate, all_coords, second_level_versions + ) + + elif other_coordinate: + verify_coordinate_in_list(other_coordinate, all_coords, first_level_versions) + else: msg = """Invalid package versioning structure.""" raise KeyError(msg) @@ -112,19 +135,35 @@ def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> No Write one dimension file for each element in the matrix (e.g. rhel7 and py3.8, rhel6 and py3.6). """ - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] + if not isinstance(matrix, dict): + raise TypeError("Matrix coordinates must be a dictionary") + + rhel_versions = matrix.get("rhel", "8") + python_versions = matrix.get("py", "311") + other_versions = None + + for k, v in matrix.items(): # find first item not rhel or py + if k not in ["rhel", "py"]: + other_versions = {k: v} + break release_base = os.path.splitext(os.path.basename(matrix_file))[0] release_folder = os.path.dirname(matrix_file) release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + + for rhel_ver, py_ver, other_ver in get_matrix( + rhel_versions, python_versions, other_versions + ): release_dict = _pick_package_versions_for_release( release_matrix, rhel_ver, py_ver, + other_ver, ) - filename = f"{format_release(release_base, rhel_ver, py_ver)}.yml" + filename = f"{format_release(release_base, rhel_ver, py_ver)}" + if other_versions: + filename = filename + f"-{other_ver}" + filename = filename + ".yml" write_to_file(release_dict, os.path.join(output_folder, filename)) @@ -134,24 +173,37 @@ def transpile_releases_for_pip( repository_file: str, matrix: dict, ) -> None: - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] + if not isinstance(matrix, dict): + raise TypeError("Matrix coordinates must be a dictionary") + + rhel_versions = matrix.get("rhel", "8") + python_versions = matrix.get("py", "311") + other_versions = None + + for k, v in matrix.items(): # find first item not rhel or py + if k not in ["rhel", "py"]: + other_versions = {k: v} + break + release_base = os.path.splitext(os.path.basename(matrix_file))[0] release_folder = os.path.dirname(matrix_file) release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") repository = load_yaml(repository_file) - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): + for rhel_ver, py_ver, other_ver in get_matrix( + rhel_versions, python_versions, other_versions + ): release_dict = _pick_package_versions_for_release( release_matrix, rhel_ver, py_ver, + other_ver, ) pip_packages = [ f"{pkg}=={version}" for pkg, version in release_dict.items() if repository[pkg][version].get("make") == "pip" ] - filename = f"{format_release(release_base, rhel_ver, py_ver)}.req" + filename = f"{format_release(release_base, rhel_ver, py_ver, other_ver)}.req" with open( os.path.join(output_folder, filename), mode="w", diff --git a/komodo/yaml_file_types.py b/komodo/yaml_file_types.py index 68746e71a..4450d925c 100644 --- a/komodo/yaml_file_types.py +++ b/komodo/yaml_file_types.py @@ -368,9 +368,9 @@ def from_yaml_string(self, value): return self def validate_upgrade_key(self, upgrade_key: str) -> None: - assert ( - upgrade_key in self.content - ), f"No section for this release ({upgrade_key}) in upgrade_proposals.yml" + assert upgrade_key in self.content, ( + f"No section for this release ({upgrade_key}) in upgrade_proposals.yml" + ) @staticmethod def validate_upgrade_proposals_file(upgrade_proposals_file_content: dict) -> None: diff --git a/tests/input/test_custom_coordinate_release.yml b/tests/input/test_custom_coordinate_release.yml new file mode 100644 index 000000000..df554265a --- /dev/null +++ b/tests/input/test_custom_coordinate_release.yml @@ -0,0 +1,26 @@ +parcel: + numpy1: 1.0.0 + numpy2: 2.0.0 +letter: + py38: + numpy1: 1.38.0 + numpy2: 2.28.0 + py311: + numpy1: 1.311.0 + numpy2: 2.211.0 +box: + rhel8: + py38: + numpy1: 1.38.8 + numpy2: 2.28.8 + py311: + numpy1: 1.311.8 + numpy2: 2.211.8 + rhel9: + py38: + numpy1: 1.38.9 + numpy2: 2.28.9 + py311: + numpy1: 1.311.9 + numpy2: 2.211.9 +case: 0.0.1 diff --git a/tests/test_matrix.py b/tests/test_matrix.py index d8120fee8..760421051 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -3,8 +3,15 @@ from komodo import matrix -def test_format_matrix(): - assert matrix.format_release("base", "rhel6", "py27") == "base-py27-rhel6" +@pytest.mark.parametrize( + ("base", "rhel", "python", "other", "expected"), + [ + ("base", "rhel6", "py27", None, "base-py27-rhel6"), + ("base", "rhel6", "py27", "numpy1", "base-py27-rhel6-numpy1"), + ], +) +def test_format_matrix(base, rhel, python, other, expected): + assert matrix.format_release(base, rhel, python, other) == expected @pytest.mark.parametrize( @@ -14,8 +21,37 @@ def test_format_matrix(): ("1970.12.rc0-foo-py38-rhel7", "1970.12.rc0-foo"), ("1970.12.03", "1970.12.03"), (matrix.format_release("1970.12.04", "rhel7", "py38"), "1970.12.04"), + (matrix.format_release("1990.06.04", "rhel9", "py38", "numpy2"), "1990.06.04"), ("1970.12.05-rhel8-py27", "1970.12.05-rhel8-py27"), # outside matrix + ("2025.02.00-py311-rhel9-numpy2", "2025.02.00"), ], ) def test_get_matrix_base(test_input, expected): assert matrix.get_matrix_base(test_input) == expected + + +@pytest.mark.parametrize( + ("rhel_ver", "py_ver", "other_ver", "expected_yield"), + [ + ( + ["8"], + ["38", "311"], + None, + [("rhel8", "py38", None), ("rhel8", "py311", None)], + ), + ( + ["8", "9"], + ["311"], + {"numpy": ["1", "2"]}, + [ + ("rhel8", "py311", "numpy1"), + ("rhel8", "py311", "numpy2"), + ("rhel9", "py311", "numpy1"), + ("rhel9", "py311", "numpy2"), + ], + ), + ], +) +def test_get_matrix(rhel_ver, py_ver, other_ver, expected_yield): + yielded = list(matrix.get_matrix(rhel_ver, py_ver, other_ver)) + assert yielded == expected_yield diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py index 183ba936a..c21a6a6a6 100644 --- a/tests/test_release_transpiler.py +++ b/tests/test_release_transpiler.py @@ -1,29 +1,27 @@ import os +from contextlib import contextmanager import pytest +import yaml from komodo.release_transpiler import ( - get_py_coords, transpile_releases, transpile_releases_for_pip, ) from tests import _get_test_root -builtins = { - "lib1": { - "rhel6": {"py27": "0.1.2", "py36": "1.2.3", "py38": "1.2.4"}, - "rhel7": { - "py27": "0.1.2+builtin", - "py36": "1.2.3+builtin", - "py38": "1.2.3+builtin", - }, - }, -} + +@contextmanager +def does_not_raise(): + yield @pytest.mark.parametrize( "matrix", - [({"py": ["3.8"], "rhel": ["7"]}), ({"py": ["3.8", "3.11"], "rhel": ["7", "8"]})], + [ + ({"py": ["3.8"], "rhel": ["7"]}), + ({"py": ["3.8", "3.11"], "rhel": ["7", "8"]}), + ], ) def test_transpile_add_argument(tmpdir, matrix): release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") @@ -39,6 +37,52 @@ def test_transpile_add_argument(tmpdir, matrix): ) +@pytest.mark.parametrize( + ("matrix", "expectation"), + [ + ({"py": ["3.8"], "rhel": ["8"], "numpy": ["1"]}, does_not_raise()), + ({"py": ["3.11"], "rhel": ["9"], "numpy": ["2"]}, does_not_raise()), + ( + {"py": ["3.8", "3.11"], "rhel": ["8", "9"], "numpy": ["1", "2"]}, + does_not_raise(), + ), + ({"py": ["3.8", "3.11"], "rhel": ["8", "9"]}, pytest.raises(KeyError)), + ( + {"py": ["3.8", "3.11"], "rhel": ["8", "9"], "numpy": ["3"]}, + pytest.raises(KeyError), + ), + ], +) +def test_transpile_custom_coordinate_releases(tmpdir, matrix, expectation): + release_file = os.path.join( + _get_test_root(), "input", "test_custom_coordinate_release.yml" + ) + release_base = os.path.basename(release_file).strip(".yml") + + packages = ["parcel", "letter", "box", "case"] + keywords = ["rhel", "py", "numpy"] + + with tmpdir.as_cwd(), expectation: + transpile_releases(release_file, os.getcwd(), matrix) + + for rhel_coordinate in matrix["rhel"]: + rhel_coordinate_filename_format = f"rhel{rhel_coordinate}" + for py_coordinate in matrix["py"]: + py_coordinate_filename_format = f"py{py_coordinate.replace('.', '')}" + for custom_coordinate in matrix["numpy"]: + custom_coordinate_filename_format = f"numpy{custom_coordinate}" + + release_file = f"{release_base}-{py_coordinate_filename_format}-{rhel_coordinate_filename_format}-{custom_coordinate_filename_format}.yml" + assert os.path.isfile(release_file) + with open(release_file, encoding="utf-8") as file: + content = yaml.safe_load(file) + + for p in packages: + assert p in content + for k in keywords: + assert k not in content + + @pytest.mark.parametrize( ("matrix", "error_message_content"), [ @@ -51,8 +95,12 @@ def test_transpile_add_argument(tmpdir, matrix): {"py": ["3.6"], "rhel": ["5"]}, ["rhel5", "lib1"], ), + ( + {"py": ["3.6"], "rhel": ["7"], "numpy": ["2"]}, + ["numpy2", "lib1"], + ), ], - ids=["Pass for all packages", "Fail", "Fail"], + ids=["Pass for all packages", "Fail", "Fail", "Missing custom numpy coordinate"], ) def test_check_version_exists_for_coordinates(matrix, error_message_content, tmpdir): release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") @@ -60,14 +108,7 @@ def test_check_version_exists_for_coordinates(matrix, error_message_content, tmp with tmpdir.as_cwd(): transpile_releases(release_file, os.getcwd(), matrix) except KeyError as exception_info: - assert all(word in str(exception_info) for word in error_message_content) - - -def test_get_py_coords(): - release_folder = os.path.join(_get_test_root(), "data", "test_releases") - release_base = "2020.01.a1" - py_coords = get_py_coords(release_base, release_folder) - assert py_coords == ["py27", "py36", "py38"] + assert all(word in str(exception_info.args) for word in error_message_content) def test_transpile_for_pip(tmpdir): diff --git a/tests/test_switch.py b/tests/test_switch.py index 57f531900..c1e9599f9 100644 --- a/tests/test_switch.py +++ b/tests/test_switch.py @@ -19,10 +19,10 @@ def test_write_activator_switches(tmpdir): == f""" if [[ $(uname -r) == *el7* ]] ; then # Get the full path of the sourced script - { bash_source_script_path } + {bash_source_script_path} if [[ $script_path == *deprecated-rhel7* ]] ; then - export KOMODO_ROOT={ prefix } - KOMODO_RELEASE_REAL={ expected_release } + export KOMODO_ROOT={prefix} + KOMODO_RELEASE_REAL={expected_release} source $KOMODO_ROOT/$KOMODO_RELEASE_REAL-rhel7/enable export PS1="(${{KOMODO_RELEASE_REAL}}) ${{_PRE_KOMODO_PS1}}"