Update transpiling to also cover an optional extra argument

.github/workflows/test.yml

@@ -6,6 +6,7 @@ jobs:
   build:
     env:
       RELEASE_NAME: release-matrix
+      RELEASE_FULL_NAME: release-matrix.yml
       REPOSITORY: repository.yml
 
     strategy:
@@ -42,19 +43,19 @@ jobs:
 
     - name: Lint examples
       run: |
-        komodo-lint examples/stable.yml examples/repository.yml
+        komodo-lint examples/stable.yml examples/${{env.REPOSITORY}}
 
     - name: Transpile file
       run: |
         komodo-transpiler transpile \
-            --matrix-file="ci/${{env.RELEASE_NAME}}.yml" \
+            --matrix-file=ci/"${{ env.RELEASE_FULL_NAME }}" \
             --output ci --matrix-coordinates "{rhel: ['7'], py: ['${{matrix.python-version}}']}"
-            
+
     - name: Full integration test
       run: |
         py_version_number=$(echo "${{ matrix.python-version }}" | sed 's/\.//g')
         kmd \
-            ci/${{env.RELEASE_NAME}}-py$py_version_number-rhel7.yml \
+            ci/${{ env.RELEASE_NAME }}-py$py_version_number-rhel7.yml \
             ci/${{env.REPOSITORY}} \
             --workspace ${{ runner.temp }}/kmd-ws \
             --prefix ${{ runner.temp }}/prefix \

komodo/check_up_to_date_pypi.py

@@ -182,9 +182,7 @@ def run_check_up_to_date(
     release_file,
     repository_file,
     python_version=(
-        f"{sys.version_info.major}."
-        f"{sys.version_info.minor}."
-        f"{sys.version_info.micro}"
+        f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
     ),
     propose_upgrade=False,
     ignore=None,

komodo/cli.py

@@ -232,7 +232,7 @@ def rsync_komodo_to_destination(release_name: str, destination: str) -> None:
 
 def move_old_release_from_release_path_if_exists(release_path: Path) -> None:
     if release_path.exists():
-        shell(f"mv {str(release_path)} " f"{str(release_path)}.delete-{uuid.uuid4()}")
+        shell(f"mv {str(release_path)} {str(release_path)}.delete-{uuid.uuid4()}")
 
 
 def move_new_release_to_release_path(args: KomodoNamespace, release_path: Path) -> None:

komodo/matrix.py

@@ -6,35 +6,53 @@
 
 import itertools
 import re
-from typing import Iterator, Sequence, Tuple
+from typing import Dict, Iterator, Optional, Sequence, Tuple
 
 
 def get_matrix(
     rhel_versions: Sequence[str],
     py_versions: Sequence[str],
-) -> Iterator[Tuple[str, str]]:
-    """Return tuples of rhel version and Python version, representing the
-    current release matrix.
+    custom_coordinate: Optional[Dict[str, Sequence[str]]] = None,
+) -> Iterator[Tuple[str, str, str]]:
+    """Return tuples of rhel version, Python version and a single optional custom_coordinate,
+    representing the current release matrix.
     """
-    for product in itertools.product(rhel_versions, py_versions):
-        rh_ver, py_ver = product
-        yield (f"rhel{rh_ver}", f"py{str(py_ver).replace('.', '')}")
+    component_name = ""
+    component_seq = [None]
 
+    if custom_coordinate:
+        if len(custom_coordinate) != 1:
+            raise ValueError("custom_coordinate must contain exactly one item")
+        component_name, component_seq = next(iter(custom_coordinate.items()))  # pylint: disable=stop-iteration-return
 
-def format_release(base: str, rhel_ver: str, py_ver: str) -> str:
+    for product in itertools.product(rhel_versions, py_versions, component_seq):
+        rh_ver, py_ver, other_ver = product
+        rhel_tag, py_tag, other_tag = (
+            f"rhel{rh_ver}",
+            f"py{str(py_ver).replace('.', '')}",
+            f"{component_name}{other_ver}" if other_ver else None,
+        )
+
+        yield rhel_tag, py_tag, other_tag
+
+
+def format_release(
+    base: str, rhel_ver: str, py_ver: str, other_component: Optional[str] = None
+) -> str:
     """Format a base (e.g. a matrix file without the .yml suffix) such that it
     looks like a concrete release.
     """
-    return f"{base}-{py_ver}-{rhel_ver}"
+    return (
+        f"{base}-{py_ver}-{rhel_ver}-{other_component}"
+        if other_component
+        else f"{base}-{py_ver}-{rhel_ver}"
+    )
 
 
 def get_matrix_base(release_name: str) -> str:
-    """Return the base (e.g. matrix part of a concrete release). Should be the
-    inverse of format_release for actual, concrete matrix releases.
-    Hard-coded the suffix pattern '-py..-rhel.' or '-py...-rhel.'.
+    """Return the base (e.g. matrix part of a concrete release).
+    Match release name on -py[nno]-rhel[n] and delimit using that
     """
-    suffix = format_release("", "rhel[0-9]", r"py\d{2,3}")
-    if re.search(suffix, release_name):
-        return re.split(suffix, release_name)[0]
-    # no matrix suffix at all
+    if re.search(r"-py\d{2,3}-rhel[0-9]", release_name):
+        return release_name.split("-py")[0]
     return release_name

komodo/release_transpiler.py

@@ -2,35 +2,20 @@
 
 import argparse
 import os
-from typing import Dict, Sequence, Union
+import re
+from typing import Dict, Optional, Sequence, Union
 
 import yaml
 
 from komodo.matrix import format_release, get_matrix
 from komodo.prettier import load_yaml, write_to_file
 
 
-def get_py_coords(release_base: str, release_folder: str) -> Sequence[str]:
-    """Get python versions of release files inside a given release_folder."""
-    filenames_with_prefix = sorted(
-        [
-            filename
-            for filename in os.listdir(release_folder)
-            if filename.startswith(release_base)
-        ],
-    )
-    len_release_base = len(release_base + "-")
-    irrelevant_suffix_length = len(".yml")
-    return [
-        filename[len_release_base:-irrelevant_suffix_length]
-        for filename in filenames_with_prefix
-    ]
-
-
 def _pick_package_versions_for_release(
     packages: dict,
     rhel_ver: str,
     py_ver: str,
+    other_ver: Optional[str] = None,
 ) -> dict:
     """Consolidate the packages for a given combination of rhel and python version
     into a dictionary.
@@ -39,7 +24,7 @@ def _pick_package_versions_for_release(
     for pkg_name, versions in packages.items():
         version = None
         try:
-            _check_version_exists_for_coordinates(versions, rhel_ver, py_ver)
+            _check_version_exists_for_coordinates(versions, rhel_ver, py_ver, other_ver)
         except KeyError as err:
             error_msg = f"{err!s}. Failed for {pkg_name}."
             raise KeyError(error_msg) from None
@@ -48,6 +33,12 @@ def _pick_package_versions_for_release(
                 version = versions[rhel_ver][py_ver]
             elif py_ver in versions:
                 version = versions[py_ver]
+
+            if other_ver:
+                if other_ver in versions:
+                    version = versions[other_ver]
+                elif other_ver and other_ver in version:
+                    version = version[other_ver]
         else:
             version = versions
         if version:
@@ -59,6 +50,7 @@ def _check_version_exists_for_coordinates(
     pkg_versions: Union[dict, str],
     rhel_coordinate: str,
     py_coordinate: str,
+    other_coordinate: Optional[str],
 ) -> None:
     """Check the coordinates `rhel_ver` and `py_ver` input as arguments to
     build a release against the release matrix file. Raise exceptions if
@@ -79,27 +71,58 @@ def _check_version_exists_for_coordinates(
         }
         or:
         {1.1.1}.
-
+        or:
+        {
+            py36: 1.1.1, # first level
+                numpy1: 1.2.6
+                numpy2: 2.2.1
+            py38: 2.1.1,
+                numpy1: 1.2.6
+                numpy2: 2.2.1
+        }
+        or:
+        {
+            numpy2: 2.2.1
+            numpy1: 1.2.6
+        }
     """
     if isinstance(pkg_versions, str):
         return None
     first_level_versions = list(pkg_versions)
+
+    def verify_coordinate_in_list(
+        coordinate: str, all_coordinates: Sequence[str], seq: Sequence[str]
+    ) -> None:
+        if not coordinate in seq:
+            raise KeyError(
+                f"Matrix coordinate {coordinate}, part of {all_coordinates}, not found in {seq}"
+            )
+
+    all_coords = [rhel_coordinate, py_coordinate, other_coordinate]
+
     if "rhel" in first_level_versions[0]:
-        # Both rhel and python versions can have different versions
-        if rhel_coordinate not in first_level_versions:
-            msg = f"Rhel version {rhel_coordinate} not found."
-            raise KeyError(msg)
+        verify_coordinate_in_list(rhel_coordinate, all_coords, first_level_versions)
         second_level_versions = list(pkg_versions[rhel_coordinate])
-        if py_coordinate not in second_level_versions:
-            msg = f"Python version {py_coordinate} not found for rhel version {rhel_coordinate}."
-            raise KeyError(
-                msg,
+        verify_coordinate_in_list(py_coordinate, all_coords, second_level_versions)
+
+        if other_coordinate:
+            third_level_versions = list(pkg_versions[rhel_coordinate][py_coordinate])
+            verify_coordinate_in_list(
+                other_coordinate, all_coords, third_level_versions
             )
-    elif "py" in first_level_versions[0]:
-        if py_coordinate not in first_level_versions:
-            # Only python has different versions
-            msg = f"Python version {py_coordinate} not found."
-            raise KeyError(msg)
+
+    elif re.match(r"py\d{2,3}", first_level_versions[0]):
+        verify_coordinate_in_list(py_coordinate, all_coords, first_level_versions)
+
+        if other_coordinate:
+            second_level_versions = list(pkg_versions[py_coordinate])
+            verify_coordinate_in_list(
+                other_coordinate, all_coords, second_level_versions
+            )
+
+    elif other_coordinate:
+        verify_coordinate_in_list(other_coordinate, all_coords, first_level_versions)
+
     else:
         msg = """Invalid package versioning structure."""
         raise KeyError(msg)
@@ -112,19 +135,35 @@ def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> No
     Write one dimension file for each element in the matrix
     (e.g. rhel7 and py3.8, rhel6 and py3.6).
     """
-    rhel_versions = matrix["rhel"]
-    python_versions = matrix["py"]
+    if not isinstance(matrix, dict):
+        raise TypeError("Matrix coordinates must be a dictionary")
+
+    rhel_versions = matrix.get("rhel", "8")
+    python_versions = matrix.get("py", "311")
+    other_versions = None
+
+    for k, v in matrix.items():  # find first item not rhel or py
+        if k not in ["rhel", "py"]:
+            other_versions = {k: v}
+            break
 
     release_base = os.path.splitext(os.path.basename(matrix_file))[0]
     release_folder = os.path.dirname(matrix_file)
     release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml")
-    for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions):
+
+    for rhel_ver, py_ver, other_ver in get_matrix(
+        rhel_versions, python_versions, other_versions
+    ):
         release_dict = _pick_package_versions_for_release(
             release_matrix,
             rhel_ver,
             py_ver,
+            other_ver,
         )
-        filename = f"{format_release(release_base, rhel_ver, py_ver)}.yml"
+        filename = f"{format_release(release_base, rhel_ver, py_ver)}"
+        if other_versions:
+            filename = filename + f"-{other_ver}"
+        filename = filename + ".yml"
         write_to_file(release_dict, os.path.join(output_folder, filename))
 
 
@@ -134,24 +173,37 @@ def transpile_releases_for_pip(
     repository_file: str,
     matrix: dict,
 ) -> None:
-    rhel_versions = matrix["rhel"]
-    python_versions = matrix["py"]
+    if not isinstance(matrix, dict):
+        raise TypeError("Matrix coordinates must be a dictionary")
+
+    rhel_versions = matrix.get("rhel", "8")
+    python_versions = matrix.get("py", "311")
+    other_versions = None
+
+    for k, v in matrix.items():  # find first item not rhel or py
+        if k not in ["rhel", "py"]:
+            other_versions = {k: v}
+            break
+
     release_base = os.path.splitext(os.path.basename(matrix_file))[0]
     release_folder = os.path.dirname(matrix_file)
     release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml")
     repository = load_yaml(repository_file)
-    for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions):
+    for rhel_ver, py_ver, other_ver in get_matrix(
+        rhel_versions, python_versions, other_versions
+    ):
         release_dict = _pick_package_versions_for_release(
             release_matrix,
             rhel_ver,
             py_ver,
+            other_ver,
         )
         pip_packages = [
             f"{pkg}=={version}"
             for pkg, version in release_dict.items()
             if repository[pkg][version].get("make") == "pip"
         ]
-        filename = f"{format_release(release_base, rhel_ver, py_ver)}.req"
+        filename = f"{format_release(release_base, rhel_ver, py_ver, other_ver)}.req"
         with open(
             os.path.join(output_folder, filename),
             mode="w",

komodo/yaml_file_types.py

@@ -368,9 +368,9 @@ def from_yaml_string(self, value):
         return self
 
     def validate_upgrade_key(self, upgrade_key: str) -> None:
-        assert (
-            upgrade_key in self.content
-        ), f"No section for this release ({upgrade_key}) in upgrade_proposals.yml"
+        assert upgrade_key in self.content, (
+            f"No section for this release ({upgrade_key}) in upgrade_proposals.yml"
+        )
 
     @staticmethod
     def validate_upgrade_proposals_file(upgrade_proposals_file_content: dict) -> None:

tests/input/test_custom_coordinate_release.yml

@@ -0,0 +1,26 @@
+parcel:
+  numpy1: 1.0.0
+  numpy2: 2.0.0
+letter:
+  py38:
+    numpy1: 1.38.0
+    numpy2: 2.28.0
+  py311:
+    numpy1: 1.311.0
+    numpy2: 2.211.0
+box:
+  rhel8:
+    py38:
+      numpy1: 1.38.8
+      numpy2: 2.28.8
+    py311:
+      numpy1: 1.311.8
+      numpy2: 2.211.8
+  rhel9:
+    py38:
+      numpy1: 1.38.9
+      numpy2: 2.28.9
+    py311:
+      numpy1: 1.311.9
+      numpy2: 2.211.9
+case: 0.0.1