From f43d89b0346e9169ad708f7a032e579a15b6a9b6 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 5 Dec 2024 14:58:05 -0800 Subject: [PATCH 1/2] when chaining against chained envs, get upstreams+envrepo's --- .../jcsda-emc/spack-stack/stack/stack_env.py | 31 ++++++++--- .../spack-stack/tests/test_stack_create.py | 52 +++++++++++++++++-- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 83757629a..d98fbe169 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -207,6 +207,16 @@ def _copy_site_includes(self): destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) + def get_upstream_realpaths(self, upstream_path): + spack_yaml_path = os.path.realpath(os.path.join(upstream_path, "../spack.yaml")) + with open(spack_yaml_path, "r") as f: + spack_yaml = syaml.load_config(f) + upstream_paths = [upstream_path] + if "upstreams" in spack_yaml["spack"].keys(): + entries = spack_yaml["spack"]["upstreams"] + for entry in entries.items(): + upstream_paths += self.get_upstream_realpaths(entry[1]["install_tree"]) + return upstream_paths def write(self): """Write environment out to a spack.yaml in /. @@ -280,10 +290,10 @@ def write(self): "common": os.path.join(self.env_dir(), "common"), "site": os.path.join(self.env_dir(), "site"), } - for upstream_path in self.upstreams: - upstream_path = upstream_path[0] - # spack doesn't handle "~/" correctly, this fixes it: - upstream_path = os.path.expanduser(upstream_path) + all_upstreams = [] + for upstream in self.upstreams: + all_upstreams.extend(self.get_upstream_realpaths(upstream[0])) + for upstream_path in all_upstreams: if not os.path.basename(os.path.normpath(upstream_path)) == "install": logging.warning( "WARNING: Upstream path '%s' is not an 'install' directory!" @@ -298,7 +308,7 @@ def write(self): if path_parts: name = path_parts["spack_stack_ver"] + "-" + path_parts["env_name"] else: - name = os.path.basename(upstream_path) + name = os.path.realpath(os.path.join(upstream_path, "..")) upstream = "upstreams:%s:install_tree:'%s'" % (name, upstream_path) logging.info("Adding upstream path '%s'" % upstream_path) spack.config.add(upstream, scope=env_scope) @@ -319,12 +329,20 @@ def write(self): f"'{upstream_path}' do not match! Verify that you are using the same " "version of spack-stack, or really, really know what you are doing.") ) + new_envrepo = os.path.join(self.env_dir(), "envrepo") + for upstream_path in all_upstreams[::-1]: + envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo")) + if os.path.isdir(envrepo_path): + shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True) + if os.path.isdir(new_envrepo): + repo_cfg = "repos:[$env/envrepo]" + spack.config.add(repo_cfg, scope=env_scope) if self.modifypkg: logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg)) env_repo_path = os.path.join(env_dir, "envrepo") env_pkgs_path = os.path.join(env_repo_path, "packages") - os.makedirs(env_pkgs_path, exist_ok=False) + os.makedirs(env_pkgs_path, exist_ok=True) with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f: f.write("repo:\n namespace: envrepo") repo_paths = spack.config.get("repos") @@ -338,6 +356,7 @@ def write(self): pkg_path, os.path.join(env_pkgs_path, pkg_name), ignore=shutil.ignore_patterns("__pycache__"), + dirs_exist_ok=True, ) pkg_found = True # Use the first repo where the package exists: diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py index 461a912a4..64846bda1 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py @@ -179,29 +179,73 @@ def test_compilers(): @pytest.mark.extension("stack") @pytest.mark.filterwarnings("ignore::UserWarning") def test_upstream(): + base_env = os.path.join(test_dir, "base_env/install/") + os.makedirs(os.path.join(base_env, ".spack-db/"), exist_ok=True) + base_env_spack_yaml_path = os.path.realpath(os.path.join(base_env, "../spack.yaml")) + f_base_env = open(base_env_spack_yaml_path, "w") + f_base_env.write("spack:\n dummytag: dummyvalue") + f_base_env.close() stack_create( "create", "env", "--site", "hera", "--name", - "upstream_test", + "chainedA", "--dir", test_dir, "--compiler", "gcc", "--upstream", - "/test/path/to/upstream/env", + base_env, ) - spack_yaml_path = os.path.join(test_dir, "upstream_test", "spack.yaml") + spack_yaml_path = os.path.join(test_dir, "chainedA", "spack.yaml") with open(spack_yaml_path, "r") as f: spack_yaml_txt = f.read() - assert "install_tree: /test/path/to/upstream/env" in spack_yaml_txt + assert f"install_tree: {base_env}" in spack_yaml_txt assert ( "repos: [$env/envrepo]" not in spack_yaml_txt ), "--modify-pkg functionality modified spack.yaml without being called" +@pytest.mark.extension("stack") +@pytest.mark.filterwarnings("ignore::UserWarning") +def test_layered_upstreams(): + os.makedirs(os.path.join(test_dir, "chainedA/install/.spack-db/")) + os.makedirs(os.path.join(test_dir, "base_env/envrepo/")) + os.makedirs(os.path.join(test_dir, "chainedA/envrepo/")) + f_base_env_envrepo_file_path = os.path.join(test_dir, "base_env/envrepo/file") + f_chainedA_envrepo_file_path = os.path.join(test_dir, "chainedA/envrepo/file") + f_base_env_envrepo_file = open(f_base_env_envrepo_file_path, "w") + f_chainedA_envrepo_file = open(f_chainedA_envrepo_file_path, "w") + f_base_env_envrepo_file.write("bad") + f_chainedA_envrepo_file.write("good") + f_base_env_envrepo_file.close() + f_chainedA_envrepo_file.close() + stack_create( + "create", + "env", + "--site", + "hera", + "--name", + "chainedB", + "--dir", + test_dir, + "--compiler", + "gcc", + "--upstream", + os.path.join(test_dir, "chainedA/install/") + ) + spack_yaml_path = os.path.join(test_dir, "chainedB", "spack.yaml") + with open(spack_yaml_path, "r") as f: + spack_yaml_txt = f.read() + assert "/base_env/install/" in spack_yaml_txt + assert "/chainedA/install/" in spack_yaml_txt + file_path = os.path.join(test_dir, "chainedB/envrepo/file") + with open(file_path, "r") as f: + assert "good" in f.read() + + @pytest.mark.extension("stack") @pytest.mark.filterwarnings("ignore::UserWarning") def test_modifypkg(): From d52aec5f089d75dbddbddbf7b6f61f740396794a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Fri, 6 Dec 2024 08:24:47 -0800 Subject: [PATCH 2/2] fix workflows/ubuntu-ci-x86_64-gnu.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 1a73ad69b..c1d29712d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -196,7 +196,7 @@ jobs: echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt cnt=$(grep -c "WARNING.*do not match" stderr.txt || true) - if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi + if [ $cnt -lt 3 ]; then echo "Missing 'create env' warnings"; exit 1; fi - name: test-env run: |