Merge branch 'feature/ecmwf-libraries-upgrade' of https://github.com/…

…stiggy87/spack-stack into feature/ecmwf-libraries-upgrade

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -196,7 +196,7 @@ jobs:
           echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml
           spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt
           cnt=$(grep -c "WARNING.*do not match" stderr.txt || true)
-          if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi
+          if [ $cnt -lt 3 ]; then echo "Missing 'create env' warnings"; exit 1; fi
 
       - name: test-env
         run: |

configs/common/packages.yaml

@@ -22,7 +22,7 @@ packages:
   bufr:
     require: '@12.1.0 +python'
   bufr-query:
-    require: '@0.0.2 +python'
+    require: '@0.0.4 +python'
   cairo:
     require: '+pic'
   cdo:

configs/containers/specs/jedi-ci.yaml

@@ -8,7 +8,7 @@
     [email protected],
     [email protected],
     [email protected],
-    [email protected].2,
+    [email protected].4,
     [email protected],
     [email protected],
     [email protected],
@@ -24,7 +24,7 @@
     [email protected],
     [email protected],
     [email protected],
-    ip@5.0.0,
+    ip@5.1.0,
     [email protected],
     [email protected],
     [email protected],

configs/sites/tier2/blackpearl/packages_oneapi.yaml

@@ -3,7 +3,6 @@ packages:
     compiler:: [[email protected],[email protected]]
     providers:
       mpi:: [intel-oneapi-mpi]
-
   mpi:
     buildable: False
   intel-oneapi-mpi:

configs/sites/tier2/bounty/README.md

@@ -1 +1 @@
-Bounty is one of @climbfuji's development systems (Dell Laptop running Ubuntu Linux 22.04 under Windows WSL2).
+Bounty is one of @climbfuji's development systems (Dell Laptop running Alma Linux 9 under Windows WSL2).

configs/sites/tier2/bounty/compilers.yaml

@@ -1,26 +1,39 @@
 compilers:
 - compiler:
-    spec: gcc@=11.4.0
+    spec: gcc@=11.5.0
     paths:
       cc: /usr/bin/gcc
       cxx: /usr/bin/g++
       f77: /usr/bin/gfortran
       fc: /usr/bin/gfortran
     flags: {}
-    operating_system: ubuntu22.04
+    operating_system: almalinux9
     target: x86_64
     modules: []
     environment: {}
     extra_rpaths: []
 - compiler:
-    spec: gcc@=12.3.0
+    spec: gcc@=13.3.1
     paths:
-      cc: /usr/bin/gcc-12
-      cxx: /usr/bin/g++-12
-      f77: /usr/bin/gfortran-12
-      fc: /usr/bin/gfortran-12
+      cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc
+      cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++
+      f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran
+      fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran
     flags: {}
-    operating_system: ubuntu22.04
+    operating_system: almalinux9
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: oneapi@=2025.0.0
+    paths:
+      cc: /opt/intel/oneapi/compiler/2025.0/bin/icx
+      cxx: /opt/intel/oneapi/compiler/2025.0/bin/icpx
+      f77: /opt/intel/oneapi/compiler/2025.0/bin/ifx
+      fc: /opt/intel/oneapi/compiler/2025.0/bin/ifx
+    flags: {}
+    operating_system: almalinux9
     target: x86_64
     modules: []
     environment: {}

configs/sites/tier2/bounty/packages.yaml

@@ -1,23 +1,31 @@
 packages:
   autoconf:
     externals:
-    - spec: autoconf@2.71
+    - spec: autoconf@2.69
       prefix: /usr
   automake:
     externals:
-    - spec: [email protected].5
+    - spec: [email protected].2
       prefix: /usr
   binutils:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]~gold+headers
+      prefix: /usr
+  bison:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  cmake:
+    externals:
+    - spec: [email protected]
       prefix: /usr
   coreutils:
     externals:
     - spec: [email protected]
       prefix: /usr
   diffutils:
     externals:
-    - spec: diffutils@3.8
+    - spec: diffutils@3.7
       prefix: /usr
   findutils:
     externals:
@@ -33,11 +41,11 @@ packages:
       prefix: /usr
   git:
     externals:
-    - spec: git@2.34.1~tcltk
+    - spec: git@2.43.5~tcltk
       prefix: /usr
   git-lfs:
     externals:
-    - spec: git-lfs@3.0.2
+    - spec: git-lfs@3.4.1
       prefix: /usr
   gmake:
     externals:
@@ -49,12 +57,25 @@ packages:
       prefix: /usr
   m4:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
+      prefix: /usr
+  openssh:
+    externals:
+    - spec: [email protected]
       prefix: /usr
   perl:
     externals:
-    - spec: perl@5.34.0~cpanm+opcode+open+shared+threads
+    - spec: perl@5.32.1~cpanm~opcode+open+shared+threads
       prefix: /usr
+  pkgconf:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  qt:
+    buildable: False
+    externals:
+    - spec: [email protected]
+      prefix: /usr/lib64/qt5
   sed:
     externals:
     - spec: [email protected]
@@ -65,9 +86,13 @@ packages:
       prefix: /usr
   texlive:
     externals:
-    - spec: texlive@20220321
+    - spec: texlive@20200406
       prefix: /usr
   wget:
     externals:
-    - spec: [email protected]
+    - spec: [email protected]
+      prefix: /usr
+  zlib:
+    externals:
+    - spec: [email protected]
       prefix: /usr

configs/sites/tier2/bounty/packages_gcc.yaml

@@ -1,5 +1,5 @@
 packages:
   all:
-    compiler:: [gcc@12.3.0]
+    compiler:: [gcc@13.3.1]
     providers:
       mpi:: [[email protected]]

configs/sites/tier2/bounty/packages_oneapi.yaml

@@ -0,0 +1,19 @@
+packages:
+  all:
+    compiler:: [[email protected], [email protected]]
+    providers:
+      mpi:: [intel-oneapi-mpi]
+  mpi:
+    buildable: False
+  intel-oneapi-mpi:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi
+  intel-oneapi-runtime:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -207,6 +207,16 @@ def _copy_site_includes(self):
         destination = os.path.join(env_site_dir, "packages.yaml")
         self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination)
 
+    def get_upstream_realpaths(self, upstream_path):
+        spack_yaml_path = os.path.realpath(os.path.join(upstream_path, "../spack.yaml"))
+        with open(spack_yaml_path, "r") as f:
+            spack_yaml = syaml.load_config(f)
+        upstream_paths = [upstream_path]
+        if "upstreams" in spack_yaml["spack"].keys():
+            entries = spack_yaml["spack"]["upstreams"]
+            for entry in entries.items():
+                upstream_paths += self.get_upstream_realpaths(entry[1]["install_tree"])
+        return upstream_paths
 
     def write(self):
         """Write environment out to a spack.yaml in <env_dir>/<name>.
@@ -280,10 +290,10 @@ def write(self):
                 "common": os.path.join(self.env_dir(), "common"),
                 "site": os.path.join(self.env_dir(), "site"),
             }
-            for upstream_path in self.upstreams:
-                upstream_path = upstream_path[0]
-                # spack doesn't handle "~/" correctly, this fixes it:
-                upstream_path = os.path.expanduser(upstream_path)
+            all_upstreams = []
+            for upstream in self.upstreams:
+                all_upstreams.extend(self.get_upstream_realpaths(upstream[0]))
+            for upstream_path in all_upstreams:
                 if not os.path.basename(os.path.normpath(upstream_path)) == "install":
                     logging.warning(
                         "WARNING: Upstream path '%s' is not an 'install' directory!"
@@ -298,7 +308,7 @@ def write(self):
                 if path_parts:
                     name = path_parts["spack_stack_ver"] + "-" + path_parts["env_name"]
                 else:
-                    name = os.path.basename(upstream_path)
+                    name = os.path.realpath(os.path.join(upstream_path, ".."))
                 upstream = "upstreams:%s:install_tree:'%s'" % (name, upstream_path)
                 logging.info("Adding upstream path '%s'" % upstream_path)
                 spack.config.add(upstream, scope=env_scope)
@@ -319,12 +329,20 @@ def write(self):
                              f"'{upstream_path}' do not match! Verify that you are using the same "
                               "version of spack-stack, or really, really know what you are doing.")
                         )
+            new_envrepo = os.path.join(self.env_dir(), "envrepo")
+            for upstream_path in all_upstreams[::-1]:
+                envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo"))
+                if os.path.isdir(envrepo_path):
+                    shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True)
+            if os.path.isdir(new_envrepo):
+                repo_cfg = "repos:[$env/envrepo]"
+                spack.config.add(repo_cfg, scope=env_scope)
 
         if self.modifypkg:
             logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg))
             env_repo_path = os.path.join(env_dir, "envrepo")
             env_pkgs_path = os.path.join(env_repo_path, "packages")
-            os.makedirs(env_pkgs_path, exist_ok=False)
+            os.makedirs(env_pkgs_path, exist_ok=True)
             with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f:
                 f.write("repo:\n  namespace: envrepo")
             repo_paths = spack.config.get("repos")
@@ -338,6 +356,7 @@ def write(self):
                             pkg_path,
                             os.path.join(env_pkgs_path, pkg_name),
                             ignore=shutil.ignore_patterns("__pycache__"),
+                            dirs_exist_ok=True,
                         )
                         pkg_found = True
                         # Use the first repo where the package exists:

spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py

@@ -179,29 +179,73 @@ def test_compilers():
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_upstream():
+    base_env = os.path.join(test_dir, "base_env/install/")
+    os.makedirs(os.path.join(base_env, ".spack-db/"), exist_ok=True)
+    base_env_spack_yaml_path = os.path.realpath(os.path.join(base_env, "../spack.yaml"))
+    f_base_env = open(base_env_spack_yaml_path, "w")
+    f_base_env.write("spack:\n  dummytag: dummyvalue")
+    f_base_env.close()
     stack_create(
         "create",
         "env",
         "--site",
         "hera",
         "--name",
-        "upstream_test",
+        "chainedA",
         "--dir",
         test_dir,
         "--compiler",
         "gcc",
         "--upstream",
-        "/test/path/to/upstream/env",
+        base_env,
     )
-    spack_yaml_path = os.path.join(test_dir, "upstream_test", "spack.yaml")
+    spack_yaml_path = os.path.join(test_dir, "chainedA", "spack.yaml")
     with open(spack_yaml_path, "r") as f:
         spack_yaml_txt = f.read()
-    assert "install_tree: /test/path/to/upstream/env" in spack_yaml_txt
+    assert f"install_tree: {base_env}" in spack_yaml_txt
     assert (
         "repos: [$env/envrepo]" not in spack_yaml_txt
     ), "--modify-pkg functionality modified spack.yaml without being called"
 
 
+@pytest.mark.extension("stack")
+@pytest.mark.filterwarnings("ignore::UserWarning")
+def test_layered_upstreams():
+    os.makedirs(os.path.join(test_dir, "chainedA/install/.spack-db/"))
+    os.makedirs(os.path.join(test_dir, "base_env/envrepo/"))
+    os.makedirs(os.path.join(test_dir, "chainedA/envrepo/"))
+    f_base_env_envrepo_file_path = os.path.join(test_dir, "base_env/envrepo/file")
+    f_chainedA_envrepo_file_path = os.path.join(test_dir, "chainedA/envrepo/file")
+    f_base_env_envrepo_file = open(f_base_env_envrepo_file_path, "w")
+    f_chainedA_envrepo_file = open(f_chainedA_envrepo_file_path, "w")
+    f_base_env_envrepo_file.write("bad")
+    f_chainedA_envrepo_file.write("good")
+    f_base_env_envrepo_file.close()
+    f_chainedA_envrepo_file.close()
+    stack_create(
+        "create",
+        "env",
+        "--site",
+        "hera",
+        "--name",
+        "chainedB",
+        "--dir",
+        test_dir,
+        "--compiler",
+        "gcc",
+        "--upstream",
+        os.path.join(test_dir, "chainedA/install/")
+    )
+    spack_yaml_path = os.path.join(test_dir, "chainedB", "spack.yaml")
+    with open(spack_yaml_path, "r") as f:
+        spack_yaml_txt = f.read()
+    assert "/base_env/install/" in spack_yaml_txt
+    assert "/chainedA/install/" in spack_yaml_txt
+    file_path = os.path.join(test_dir, "chainedB/envrepo/file")
+    with open(file_path, "r") as f:
+        assert "good" in f.read()
+
+
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_modifypkg():