From 3567abfc71058a1798d72b2a6e0802bac061e1c7 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 30 Jul 2024 08:11:13 -0700 Subject: [PATCH 001/134] draft weekly build/test scripts --- util/weekly_build/1_DirectorySetup.sh | 5 +++++ util/weekly_build/2_GetSpackStackDevelop.sh | 6 ++++++ util/weekly_build/3_SetupUnifiedEnv.sh | 17 +++++++++++++++++ util/weekly_build/4_SpackInstall.sh | 18 ++++++++++++++++++ util/weekly_build/5_BuildCache.sh | 0 util/weekly_build/6_Cleanup.sh | 0 util/weekly_build/README | 1 + util/weekly_build/ShellSetup.sh | 16 ++++++++++++++++ .../SpackStackBuildCache_AllSteps.sh | 9 +++++++++ 9 files changed, 72 insertions(+) create mode 100644 util/weekly_build/1_DirectorySetup.sh create mode 100644 util/weekly_build/2_GetSpackStackDevelop.sh create mode 100644 util/weekly_build/3_SetupUnifiedEnv.sh create mode 100644 util/weekly_build/4_SpackInstall.sh create mode 100644 util/weekly_build/5_BuildCache.sh create mode 100644 util/weekly_build/6_Cleanup.sh create mode 100644 util/weekly_build/README create mode 100644 util/weekly_build/ShellSetup.sh create mode 100644 util/weekly_build/SpackStackBuildCache_AllSteps.sh diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/1_DirectorySetup.sh new file mode 100644 index 000000000..a73c9584f --- /dev/null +++ b/util/weekly_build/1_DirectorySetup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo Base directory: ${2?"Second arg: base directory for build"} + +mkdir -p $2 diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh new file mode 100644 index 000000000..f92e83ed7 --- /dev/null +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR +git clone https://github.com/JCSDA/spack-stack -b develop spack-stack-build-cache-${RUNID} diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh new file mode 100644 index 000000000..98830ec9d --- /dev/null +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/spack-stack-build-cache-$RUNID + +. setup.sh + +for compiler in $COMPILERS; do + spack stack create env --name build-$compiler --template unified-dev --platform $PLATFORM + cd envs/build-$compiler + sed -i "s|- compilers: \['%.*|- compilers: ['%$compiler']|" spack.yaml + # Check for duplicates and fail before doing the "real" concretization with test deps: + spack concretize --fresh 2>&1 | tee log.concretize + ${SPACK_STACK_DIR:?}/utils/show_duplicate_packages.py -d -i crtm log.concretize + spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test +done diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh new file mode 100644 index 000000000..36f49e6e7 --- /dev/null +++ b/util/weekly_build/4_SpackInstall.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/spack-stack-build-cache-${RUNID} + +. setup.sh + +installopts="--show-log-on-error" + +for compiler in $COMPILERS; do + cd envs/build-$compiler + spack env activate . + # Just install the packages we're testing (+dependencies): + spack install $installopts --test root $PACKAGESTOTEST 2>&1 | tee log.install_withtesting + # Install the rest of the stack as usual: + spack install $installopts 2>&1 | tee log.install +done diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh new file mode 100644 index 000000000..e69de29bb diff --git a/util/weekly_build/6_Cleanup.sh b/util/weekly_build/6_Cleanup.sh new file mode 100644 index 000000000..e69de29bb diff --git a/util/weekly_build/README b/util/weekly_build/README new file mode 100644 index 000000000..31232331b --- /dev/null +++ b/util/weekly_build/README @@ -0,0 +1 @@ +Concretization for deployments should be done with `spack concretize --fresh` so as to ensure that the stack would concretize the same way if the build cache weren't there. diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh new file mode 100644 index 000000000..5cfcb5f58 --- /dev/null +++ b/util/weekly_build/ShellSetup.sh @@ -0,0 +1,16 @@ +echo Run ID: ${1?"First arg: Unique run ID"} +echo Base directory: ${2?"Second arg: base directory for build"} +echo Platform name: ${3?"Third arg: platform name ('hera', 'hercules', etc.)"} + +export RUNID=$1 +export BASEDIR=$2 +export PLATFORM=$3 + +if [ ${BASEDIR::1} != "/" ]; then + echo "FATAL ERROR: Directory should be an absolute path!" + exit 1 +fi + +export COMPILERS=intel + +export PACKAGESTOTEST="libpng libaec jasper scotch w3emc bacio g2 g2c" diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh new file mode 100644 index 000000000..3b666de5c --- /dev/null +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +. ShellSetup.sh $* + +export SETUPDONE=YES + + From b475bbb719be0da51db705c2ce690b657a840efe Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:30:00 -0700 Subject: [PATCH 002/134] add testing for apps like UFSWM --- util/weekly_build/6_AppTests.sh | 9 +++++++++ util/weekly_build/6_Cleanup.sh | 0 util/weekly_build/apptests/test_ufswm.sh | 5 +++++ 3 files changed, 14 insertions(+) create mode 100755 util/weekly_build/6_AppTests.sh delete mode 100644 util/weekly_build/6_Cleanup.sh create mode 100644 util/weekly_build/apptests/test_ufswm.sh diff --git a/util/weekly_build/6_AppTests.sh b/util/weekly_build/6_AppTests.sh new file mode 100755 index 000000000..4f3d49b7d --- /dev/null +++ b/util/weekly_build/6_AppTests.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +if [ "$TEST_UFSWM" == ON ]; then + ./apptests/test_ufswm.sh +fi diff --git a/util/weekly_build/6_Cleanup.sh b/util/weekly_build/6_Cleanup.sh deleted file mode 100644 index e69de29bb..000000000 diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh new file mode 100644 index 000000000..0cfcd88ab --- /dev/null +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# Do some stuff, and exit non-zero if UFSWM cannot be successfully built and +# tested (ideally, have a discernably different error condition if there are +# numerical differences) From 0ba2377f126b80236d6dc77b93b976e79f8173ca Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:31:56 -0700 Subject: [PATCH 003/134] set executable perms --- util/weekly_build/1_DirectorySetup.sh | 0 util/weekly_build/2_GetSpackStackDevelop.sh | 0 util/weekly_build/3_SetupUnifiedEnv.sh | 0 util/weekly_build/4_SpackInstall.sh | 0 util/weekly_build/5_BuildCache.sh | 0 util/weekly_build/SpackStackBuildCache_AllSteps.sh | 0 6 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 util/weekly_build/1_DirectorySetup.sh mode change 100644 => 100755 util/weekly_build/2_GetSpackStackDevelop.sh mode change 100644 => 100755 util/weekly_build/3_SetupUnifiedEnv.sh mode change 100644 => 100755 util/weekly_build/4_SpackInstall.sh mode change 100644 => 100755 util/weekly_build/5_BuildCache.sh mode change 100644 => 100755 util/weekly_build/SpackStackBuildCache_AllSteps.sh diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/1_DirectorySetup.sh old mode 100644 new mode 100755 diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh old mode 100644 new mode 100755 diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh old mode 100644 new mode 100755 diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh old mode 100644 new mode 100755 diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh old mode 100644 new mode 100755 diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh old mode 100644 new mode 100755 From fcb6fe2705af4ddf3b8e122da6806df8ea07289b Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:32:46 -0700 Subject: [PATCH 004/134] add set -e to various scripts --- util/weekly_build/1_DirectorySetup.sh | 2 ++ util/weekly_build/2_GetSpackStackDevelop.sh | 2 ++ util/weekly_build/3_SetupUnifiedEnv.sh | 2 ++ util/weekly_build/4_SpackInstall.sh | 2 ++ 4 files changed, 8 insertions(+) diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/1_DirectorySetup.sh index a73c9584f..350e8da55 100755 --- a/util/weekly_build/1_DirectorySetup.sh +++ b/util/weekly_build/1_DirectorySetup.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + echo Base directory: ${2?"Second arg: base directory for build"} mkdir -p $2 diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index f92e83ed7..28b90bbe5 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -2,5 +2,7 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi +set -e + cd $RUNDIR git clone https://github.com/JCSDA/spack-stack -b develop spack-stack-build-cache-${RUNID} diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index 98830ec9d..60ca3a19e 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -2,6 +2,8 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi +set -e + cd $RUNDIR/spack-stack-build-cache-$RUNID . setup.sh diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 36f49e6e7..f22cce306 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi cd $RUNDIR/spack-stack-build-cache-${RUNID} From 9d2aeeeb6bdd5a6ca33b2cd787fb9a4305beeea7 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:33:05 -0700 Subject: [PATCH 005/134] git clone --recurse-submodules --- util/weekly_build/2_GetSpackStackDevelop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index 28b90bbe5..115ed4642 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -5,4 +5,4 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi set -e cd $RUNDIR -git clone https://github.com/JCSDA/spack-stack -b develop spack-stack-build-cache-${RUNID} +git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop spack-stack-build-cache-${RUNID} From 9886b5d2e0ac257f7a1db75f142193e283b61af8 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:33:39 -0700 Subject: [PATCH 006/134] various fixes/updates --- util/weekly_build/3_SetupUnifiedEnv.sh | 6 +- util/weekly_build/ShellSetup.sh | 65 +++++++++++++++++-- .../SpackStackBuildCache_AllSteps.sh | 8 ++- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index 60ca3a19e..277274fd3 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -9,11 +9,11 @@ cd $RUNDIR/spack-stack-build-cache-$RUNID . setup.sh for compiler in $COMPILERS; do - spack stack create env --name build-$compiler --template unified-dev --platform $PLATFORM + spack stack create env --name build-$compiler --template unified-dev --site $PLATFORM --compiler $compiler cd envs/build-$compiler - sed -i "s|- compilers: \['%.*|- compilers: ['%$compiler']|" spack.yaml + spack env activate . # Check for duplicates and fail before doing the "real" concretization with test deps: spack concretize --fresh 2>&1 | tee log.concretize - ${SPACK_STACK_DIR:?}/utils/show_duplicate_packages.py -d -i crtm log.concretize + ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 5cfcb5f58..6579e6eae 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -1,16 +1,73 @@ echo Run ID: ${1?"First arg: Unique run ID"} -echo Base directory: ${2?"Second arg: base directory for build"} +echo Run directory: ${2?"Second arg: base directory for build"} echo Platform name: ${3?"Third arg: platform name ('hera', 'hercules', etc.)"} export RUNID=$1 -export BASEDIR=$2 +export RUNDIR=$2 export PLATFORM=$3 -if [ ${BASEDIR::1} != "/" ]; then +if [ ${RUNDIR::1} != "/" ]; then echo "FATAL ERROR: Directory should be an absolute path!" exit 1 fi export COMPILERS=intel -export PACKAGESTOTEST="libpng libaec jasper scotch w3emc bacio g2 g2c" +export PACKAGESTOTEST="libpng libaec jasper scotch w3emc g2 g2c" + +case $PLATFORM in + hercules) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/hercules/build_cache} + ;; + orion) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/orion/build_cache} + ;; + discover16) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu16/build_cache} + ;; + discover17) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu17/build_cache} + ;; + derecho) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} + ;; + acorn) + COMPILERS="intel" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} + TEST_UFSWM=ON + SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -P NCEPLIBS-DEV -V -I --" + ;; + gaea) + COMPILERS="intel" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/ncrc/proj/epic/spack-stack/build_cache} + ;; + hera) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/scratch1/NCEPDEV/nems/role.epic/spack-stack/build_cache} + ;; + jet) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/build_cache} + ;; + narwhal) + COMPILERS="intel gcc" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} + ;; + nautilus) + COMPILERS="intel" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} + ;; + s4) + COMPILERS="intel" + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/data/prod/jedi/spack-stack/build_cache} + ;; + linux.default) + COMPILERS="gcc" +esac + +echo "Build cache target directory: ${BUILD_CACHE_DIR?'BUILD_CACHE_DIR must be set!'}" diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 3b666de5c..d0a99daa5 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -6,4 +6,10 @@ set -e export SETUPDONE=YES - +./1_DirectorySetup.sh $* +./2_GetSpackStackDevelop.sh $* +./3_SetupUnifiedEnv.sh $* +./4_SpackInstall.sh $* +./5_BuildCache.sh $* +./6_AppTests.sh $* +./7_Cleanup.sh $* From f636b599fc491b14aaa48401e667e6d4e62bfaf9 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:36:19 -0700 Subject: [PATCH 007/134] allow use of job schedulers for 'spack install' --- util/weekly_build/4_SpackInstall.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index f22cce306..dd0f5c66c 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -14,7 +14,7 @@ for compiler in $COMPILERS; do cd envs/build-$compiler spack env activate . # Just install the packages we're testing (+dependencies): - spack install $installopts --test root $PACKAGESTOTEST 2>&1 | tee log.install_withtesting + $SCHEDULER_CMD spack install $installopts --test root $PACKAGESTOTEST 2>&1 | tee log.install_withtesting # Install the rest of the stack as usual: - spack install $installopts 2>&1 | tee log.install + $SCHEDULER_CMD spack install $installopts 2>&1 | tee log.install done From 56a2615b452f3660611d91e6656b4bbd2d1a9247 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:36:30 -0700 Subject: [PATCH 008/134] add build caching script --- util/weekly_build/5_BuildCache.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh index e69de29bb..5e219febd 100755 --- a/util/weekly_build/5_BuildCache.sh +++ b/util/weekly_build/5_BuildCache.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +cd $RUNDIR/spack-stack-build-cache-${RUNID} + +. setup.sh + +for compiler in $COMPILERS; do + cd envs/build-$compiler + spack env activate . + spack buildcache push --unsigned --force ${BUILD_CACHE_DIR?"BUILD_CACHE_DIR must be set!"} +done From 2b538f6844cf498778eb50ae8a85c1942ecd7d48 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 15:36:35 -0700 Subject: [PATCH 009/134] add cleanup script --- util/weekly_build/7_Cleanup.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 util/weekly_build/7_Cleanup.sh diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/7_Cleanup.sh new file mode 100755 index 000000000..5deb09456 --- /dev/null +++ b/util/weekly_build/7_Cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +/usr/bin/rm -rf ${RUNDIR:?}/spack-stack-build-cache-${RUNID:?} From 84e0b6598e8400fb68703dad0aa5b25345dc7770 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 00:00:19 +0000 Subject: [PATCH 010/134] various updates after acorn testing --- util/weekly_build/2_GetSpackStackDevelop.sh | 2 +- util/weekly_build/3_SetupUnifiedEnv.sh | 6 +++--- util/weekly_build/4_SpackInstall.sh | 11 ++++++----- util/weekly_build/5_BuildCache.sh | 6 +++--- util/weekly_build/7_Cleanup.sh | 2 +- util/weekly_build/ShellSetup.sh | 8 +++++--- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index 115ed4642..f28c6fe84 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -5,4 +5,4 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi set -e cd $RUNDIR -git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop spack-stack-build-cache-${RUNID} +git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop ${RUNID} diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index 277274fd3..e775b8d44 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -4,13 +4,13 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi set -e -cd $RUNDIR/spack-stack-build-cache-$RUNID +cd $RUNDIR/$RUNID . setup.sh for compiler in $COMPILERS; do - spack stack create env --name build-$compiler --template unified-dev --site $PLATFORM --compiler $compiler - cd envs/build-$compiler + spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler + cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . # Check for duplicates and fail before doing the "real" concretization with test deps: spack concretize --fresh 2>&1 | tee log.concretize diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index dd0f5c66c..1d7f2d231 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -4,17 +4,18 @@ set -e if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -cd $RUNDIR/spack-stack-build-cache-${RUNID} +cd $RUNDIR/${RUNID} . setup.sh -installopts="--show-log-on-error" +INSTALL_OPTS="--show-log-on-error $INSTALL_OPTS" for compiler in $COMPILERS; do - cd envs/build-$compiler + cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . + spack fetch # Just install the packages we're testing (+dependencies): - $SCHEDULER_CMD spack install $installopts --test root $PACKAGESTOTEST 2>&1 | tee log.install_withtesting + $SCHEDULER_CMD $(which spack) install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.install_withtesting # Install the rest of the stack as usual: - $SCHEDULER_CMD spack install $installopts 2>&1 | tee log.install + $SCHEDULER_CMD $(which spack) install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install done diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh index 5e219febd..faacc86bd 100755 --- a/util/weekly_build/5_BuildCache.sh +++ b/util/weekly_build/5_BuildCache.sh @@ -4,12 +4,12 @@ set -e if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -cd $RUNDIR/spack-stack-build-cache-${RUNID} +cd $RUNDIR/$RUNID . setup.sh for compiler in $COMPILERS; do - cd envs/build-$compiler + cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack buildcache push --unsigned --force ${BUILD_CACHE_DIR?"BUILD_CACHE_DIR must be set!"} + spack buildcache push --unsigned --force ${BUILD_CACHE_DIR?"BUILD_CACHE_DIR must be set!"} $PACKAGES_TO_INSTALL done diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/7_Cleanup.sh index 5deb09456..7894356be 100755 --- a/util/weekly_build/7_Cleanup.sh +++ b/util/weekly_build/7_Cleanup.sh @@ -4,4 +4,4 @@ set -e if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -/usr/bin/rm -rf ${RUNDIR:?}/spack-stack-build-cache-${RUNID:?} +/usr/bin/rm -rf ${RUNDIR:?}/${RUNID:?} diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 6579e6eae..f87a47185 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -13,7 +13,7 @@ fi export COMPILERS=intel -export PACKAGESTOTEST="libpng libaec jasper scotch w3emc g2 g2c" +export PACKAGES_TO_TEST="libpng libaec jasper scotch w3emc g2 g2c" case $PLATFORM in hercules) @@ -37,10 +37,12 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) - COMPILERS="intel" + COMPILERS="intel@19 intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} TEST_UFSWM=ON - SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -P NCEPLIBS-DEV -V -I --" + SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l nodes=1:ppn=6 -l walltime=03:00:00 -V -Wblock=true --" + PACKAGES_TO_INSTALL=" ufs-weather-model-env global-workflow-env upp-env" + INSTALL_OPTS="-j6" ;; gaea) COMPILERS="intel" From c59abde7a0a132fc2f71e81349519311b8204c86 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 00:02:30 +0000 Subject: [PATCH 011/134] fix acorn qsub --- util/weekly_build/ShellSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index f87a47185..c2ab11ce6 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -40,7 +40,7 @@ case $PLATFORM in COMPILERS="intel@19 intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} TEST_UFSWM=ON - SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l nodes=1:ppn=6 -l walltime=03:00:00 -V -Wblock=true --" + SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" PACKAGES_TO_INSTALL=" ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" ;; From 8ebba5c5c3afa5d6a6480cf862ddd387c3bbfdba Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 00:04:46 +0000 Subject: [PATCH 012/134] allow setting compiler version with --compiler --- spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index b63dd93f1..85b701839 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -100,7 +100,7 @@ def __init__(self, **kwargs): if not self.name: # site = self.site if self.site else 'default' - self.name = "{}.{}.{}".format(self.template, self.site, self.compiler) + self.name = "{}.{}.{}".format(self.template, self.site, self.compiler.replace("@", "-")) def env_dir(self): """env_dir is /""" @@ -170,7 +170,7 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -203,7 +203,7 @@ def _copy_site_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy site package config(s) packages_yaml_path = os.path.join(env_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -260,7 +260,7 @@ def write(self): # DH I am too stupid to do this the "spack way" ... definitions = spack.config.get("definitions", scope=env_scope) if definitions: - target_compiler = f"%{self.compiler}" + target_compiler = f"%{self.compiler}".split("@")[0] for i in range(len(definitions)): if "compilers" in definitions[i]: j = len(definitions[i]["compilers"])-1 From bea08fff13e6cc49869a2e057cab315c13bc4767 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 17:07:33 -0700 Subject: [PATCH 013/134] fix typo --- util/weekly_build/ShellSetup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index c2ab11ce6..a47a98607 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -39,10 +39,10 @@ case $PLATFORM in acorn) COMPILERS="intel@19 intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} - TEST_UFSWM=ON SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" - PACKAGES_TO_INSTALL=" ufs-weather-model-env global-workflow-env upp-env" + PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" + TEST_UFSWM=ON ;; gaea) COMPILERS="intel" From a766ba34a7666bab95fa330e231eddc23ae19fc2 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 17:17:48 -0700 Subject: [PATCH 014/134] Add alerting --- util/weekly_build/ShellSetup.sh | 1 + util/weekly_build/SpackStackBuildCache_AllSteps.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index a47a98607..b23afc168 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -42,6 +42,7 @@ case $PLATFORM in SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" + ALERT_CMD="mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov hang.lei@noaa.gov <<(echo 'Weekly spack-stack build failed. Run ID: $RUNID')" TEST_UFSWM=ON ;; gaea) diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index d0a99daa5..6b27b3101 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -4,6 +4,13 @@ set -e . ShellSetup.sh $* +trap "ERROR" ERR + +function ERROR { + eval "$ALERT_CMD" + exit 1 +} + export SETUPDONE=YES ./1_DirectorySetup.sh $* From fd8496b0408acd4e171709ffab30edc5ca6de4b1 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 00:45:28 +0000 Subject: [PATCH 015/134] update ALERT_CMD logic to use function --- util/weekly_build/ShellSetup.sh | 4 +++- util/weekly_build/SpackStackBuildCache_AllSteps.sh | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index b23afc168..c28bbd6fc 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -42,7 +42,9 @@ case $PLATFORM in SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" - ALERT_CMD="mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov hang.lei@noaa.gov <<(echo 'Weekly spack-stack build failed. Run ID: $RUNID')" + function ALERT_CMD { + mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov hang.lei@noaa.gov < <(echo "Weekly spack-stack build failed. Run ID: $RUNID") + } TEST_UFSWM=ON ;; gaea) diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 6b27b3101..a17cc2959 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -6,10 +6,7 @@ set -e trap "ERROR" ERR -function ERROR { - eval "$ALERT_CMD" - exit 1 -} +function ERROR { ALERT_CMD ; exit 1;} export SETUPDONE=YES From da5557e41e62a178d6cfc3250b68cbe34fe1bfe2 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 02:05:28 +0000 Subject: [PATCH 016/134] misc. updates/fixes for cron-based runs --- util/weekly_build/1_DirectorySetup.sh | 2 +- util/weekly_build/2_GetSpackStackDevelop.sh | 6 ++++-- util/weekly_build/3_SetupUnifiedEnv.sh | 5 ++++- util/weekly_build/4_SpackInstall.sh | 4 +++- util/weekly_build/5_BuildCache.sh | 2 +- util/weekly_build/6_AppTests.sh | 2 +- util/weekly_build/ShellSetup.sh | 4 ++-- util/weekly_build/SpackStackBuildCache_AllSteps.sh | 4 +++- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/1_DirectorySetup.sh index 350e8da55..726d6e731 100755 --- a/util/weekly_build/1_DirectorySetup.sh +++ b/util/weekly_build/1_DirectorySetup.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex echo Base directory: ${2?"Second arg: base directory for build"} diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index f28c6fe84..2f60da7c8 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -2,7 +2,9 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -set -e +set -ex cd $RUNDIR -git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop ${RUNID} +if [ ! -d $RUNID ]; then + git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop $RUNID +fi diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index e775b8d44..4184fe0cb 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -2,13 +2,16 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -set -e +set -ex cd $RUNDIR/$RUNID +set +x . setup.sh +set -x for compiler in $COMPILERS; do + rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 1d7f2d231..cf0baa30c 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -1,12 +1,14 @@ #!/bin/bash -set -e +set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi cd $RUNDIR/${RUNID} +set +x . setup.sh +set -x INSTALL_OPTS="--show-log-on-error $INSTALL_OPTS" diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh index faacc86bd..40e15f94e 100755 --- a/util/weekly_build/5_BuildCache.sh +++ b/util/weekly_build/5_BuildCache.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi diff --git a/util/weekly_build/6_AppTests.sh b/util/weekly_build/6_AppTests.sh index 4f3d49b7d..3764d8a93 100755 --- a/util/weekly_build/6_AppTests.sh +++ b/util/weekly_build/6_AppTests.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index c28bbd6fc..509682cc7 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -37,13 +37,13 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) - COMPILERS="intel@19 intel@2022" + COMPILERS="intel" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" function ALERT_CMD { - mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov hang.lei@noaa.gov < <(echo "Weekly spack-stack build failed. Run ID: $RUNID") + mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed. Run ID: $RUNID") } TEST_UFSWM=ON ;; diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index a17cc2959..ac85a61ff 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -1,6 +1,8 @@ #!/bin/bash -set -e +set -ex + +cd $(dirname $0) . ShellSetup.sh $* From 7a749dca26f0763fca2539a08b0a92f87028bf55 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 02:07:31 +0000 Subject: [PATCH 017/134] user fork for now to incorporate misc. changes --- util/weekly_build/2_GetSpackStackDevelop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index 2f60da7c8..d892bd421 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -6,5 +6,5 @@ set -ex cd $RUNDIR if [ ! -d $RUNID ]; then - git clone --recurse-submodules https://github.com/JCSDA/spack-stack -b develop $RUNID + git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID fi From 2d9c5dc8d04b5b359bcd4ebdaee1f0df893c2237 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 02:08:15 +0000 Subject: [PATCH 018/134] use intel@2022 only for acorn --- util/weekly_build/ShellSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 509682cc7..70fafe3f5 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -37,7 +37,7 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) - COMPILERS="intel" + COMPILERS="intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" From 60ed58ffed413c6a60c626c44ff491853c1f0f3b Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 02:34:11 +0000 Subject: [PATCH 019/134] allow compilers with version in matrix defs --- spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index 85b701839..83757629a 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -260,14 +260,10 @@ def write(self): # DH I am too stupid to do this the "spack way" ... definitions = spack.config.get("definitions", scope=env_scope) if definitions: - target_compiler = f"%{self.compiler}".split("@")[0] + target_compiler = f"%{self.compiler}" for i in range(len(definitions)): if "compilers" in definitions[i]: - j = len(definitions[i]["compilers"])-1 - while j>=0: - if not definitions[i]["compilers"][j] == target_compiler: - definitions[i]["compilers"].pop(j) - j -= 1 + definitions[i] = {"compilers": [target_compiler]} spack.config.set("definitions", definitions, scope=env_scope) if self.install_prefix: From 703d61db62e4410f1cb68b7375623352c5d23fb7 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 19:35:40 -0700 Subject: [PATCH 020/134] misc update --- util/weekly_build/2_GetSpackStackDevelop.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index d892bd421..67e95b136 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -5,6 +5,9 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi set -ex cd $RUNDIR -if [ ! -d $RUNID ]; then +if [ -d $RUNID ]; then + cd $RUNID + git pull +else git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID fi From 5b415f49f3b5d0f0235037795e396ffcaec62246 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 19:42:52 -0700 Subject: [PATCH 021/134] remove eccodes pinned version from acorn config --- configs/sites/tier1/acorn/packages.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index 7114e61e2..bce9d88d4 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -56,10 +56,6 @@ require: - any_of: ["@1.23.1"] when: "%intel@19.1.3.304" - eccodes: - require: - - any_of: ["@2.27.0"] - when: "%intel@19.1.3.304" py-scipy: require: - any_of: ["@1.10.1"] From d0c76cf7acaf4238362fac2dc0aa94418afd5144 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 31 Jul 2024 19:55:37 -0700 Subject: [PATCH 022/134] disable concretizing with test-only deps --- util/weekly_build/3_SetupUnifiedEnv.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index 4184fe0cb..bfe08abb4 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -18,5 +18,6 @@ for compiler in $COMPILERS; do # Check for duplicates and fail before doing the "real" concretization with test deps: spack concretize --fresh 2>&1 | tee log.concretize ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf - spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test +# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. +# spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test done From ba5a37f51ab257b230a15fd9f28fdf4dc77939f5 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 1 Aug 2024 18:28:38 +0000 Subject: [PATCH 023/134] clean up shell setup/install cmd logic --- util/weekly_build/4_SpackInstall.sh | 4 ++-- util/weekly_build/ShellSetup.sh | 26 +++++++++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index cf0baa30c..4f463de2c 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -17,7 +17,7 @@ for compiler in $COMPILERS; do spack env activate . spack fetch # Just install the packages we're testing (+dependencies): - $SCHEDULER_CMD $(which spack) install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.install_withtesting + scheduler_cmd $(which spack) install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # Install the rest of the stack as usual: - $SCHEDULER_CMD $(which spack) install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install + scheduler_cmd $(which spack) install $INSTALL_OPTS $PACKAGES_TO_INSTALL done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 70fafe3f5..32cdddbb2 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -2,18 +2,22 @@ echo Run ID: ${1?"First arg: Unique run ID"} echo Run directory: ${2?"Second arg: base directory for build"} echo Platform name: ${3?"Third arg: platform name ('hera', 'hercules', etc.)"} -export RUNID=$1 -export RUNDIR=$2 -export PLATFORM=$3 +RUNID=$1 +RUNDIR=$2 +PLATFORM=$3 if [ ${RUNDIR::1} != "/" ]; then echo "FATAL ERROR: Directory should be an absolute path!" exit 1 fi -export COMPILERS=intel +COMPILERS=intel -export PACKAGES_TO_TEST="libpng libaec jasper scotch w3emc g2 g2c" +PACKAGES_TO_TEST="libpng libaec jasper scotch w3emc g2 g2c" + +function scheduler_cmd { + $* | tee -a log.install 2>&1 +} case $PLATFORM in hercules) @@ -39,10 +43,18 @@ case $PLATFORM in acorn) COMPILERS="intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} - SCHEDULER_CMD="qsub -N spack-build-cache-$RUNID -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true --" + function scheduler_cmd { + set +e + qsub -N spack-build-cache-$RUNID -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $* + rc=$? + set -e + cat spack-build-cache-${RUNID}* + return $rc + } + PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" - function ALERT_CMD { + function alert_cmd { mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed. Run ID: $RUNID") } TEST_UFSWM=ON From 4273b9d480adabf9abe6de4ac5acca8906d34929 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 1 Aug 2024 11:39:08 -0700 Subject: [PATCH 024/134] fix capitalization for alert_cmd --- util/weekly_build/SpackStackBuildCache_AllSteps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index ac85a61ff..6acc23318 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -8,7 +8,7 @@ cd $(dirname $0) trap "ERROR" ERR -function ERROR { ALERT_CMD ; exit 1;} +function ERROR { alert_cmd ; exit 1;} export SETUPDONE=YES From 6aa3b55f81ddcba4a467d5828cd9b5fbfa7a7494 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 1 Aug 2024 11:41:42 -0700 Subject: [PATCH 025/134] add generic alert_cmd --- util/weekly_build/ShellSetup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 32cdddbb2..3034b0544 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -19,6 +19,10 @@ function scheduler_cmd { $* | tee -a log.install 2>&1 } +function alert_cmd { + echo "This is a placeholder alerting function. 'alert_cmd' should be defined for each system." +} + case $PLATFORM in hercules) COMPILERS="intel gcc" From f99019765332205a4ccf5b5eaac185a0f255debe Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 1 Aug 2024 11:44:11 -0700 Subject: [PATCH 026/134] auto-export variables from ShellSetup.sh when using SpackStackBuildCache_AllSteps.sh --- util/weekly_build/SpackStackBuildCache_AllSteps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 6acc23318..b7252a71e 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -ex +set -exa cd $(dirname $0) From 285c347a93f4400d167532a1050720914a4d29a0 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 1 Aug 2024 13:21:00 -0700 Subject: [PATCH 027/134] update install logic --- util/weekly_build/4_SpackInstall.sh | 6 +++--- util/weekly_build/ShellSetup.sh | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 4f463de2c..d550421ef 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -10,14 +10,14 @@ set +x . setup.sh set -x -INSTALL_OPTS="--show-log-on-error $INSTALL_OPTS" +INSTALL_OPTS="--show-log-on-error --fail-fast $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . spack fetch # Just install the packages we're testing (+dependencies): - scheduler_cmd $(which spack) install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # Install the rest of the stack as usual: - scheduler_cmd $(which spack) install $INSTALL_OPTS $PACKAGES_TO_INSTALL + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 3034b0544..d7f9c1ca0 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -15,14 +15,14 @@ COMPILERS=intel PACKAGES_TO_TEST="libpng libaec jasper scotch w3emc g2 g2c" -function scheduler_cmd { - $* | tee -a log.install 2>&1 -} - function alert_cmd { echo "This is a placeholder alerting function. 'alert_cmd' should be defined for each system." } +function spack_install_exe { + spack $* | tee -a log.install 2>&1 +} + case $PLATFORM in hercules) COMPILERS="intel gcc" @@ -47,9 +47,11 @@ case $PLATFORM in acorn) COMPILERS="intel@2022" BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} - function scheduler_cmd { + function spack_install_exe { set +e - qsub -N spack-build-cache-$RUNID -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $* + ( qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & + ( qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & + wait rc=$? set -e cat spack-build-cache-${RUNID}* From 1cf9b581777bad4c06187dd59ef4470a47aab75c Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Thu, 1 Aug 2024 13:21:09 -0700 Subject: [PATCH 028/134] fix 5_BuildCache.sh/setup.sh --- util/weekly_build/5_BuildCache.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/5_BuildCache.sh index 40e15f94e..45982ff01 100755 --- a/util/weekly_build/5_BuildCache.sh +++ b/util/weekly_build/5_BuildCache.sh @@ -6,7 +6,9 @@ if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi cd $RUNDIR/$RUNID +set +x . setup.sh +set -x for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} From c61ca6d7c6fe0e443567048e717fccd58c675df3 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 7 Aug 2024 12:14:43 -0700 Subject: [PATCH 029/134] make COMPILERS and PACKAGES_TO_TEST overridable --- util/weekly_build/ShellSetup.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index d7f9c1ca0..f5ea86a26 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -11,9 +11,7 @@ if [ ${RUNDIR::1} != "/" ]; then exit 1 fi -COMPILERS=intel - -PACKAGES_TO_TEST="libpng libaec jasper scotch w3emc g2 g2c" +PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} function alert_cmd { echo "This is a placeholder alerting function. 'alert_cmd' should be defined for each system." @@ -25,27 +23,27 @@ function spack_install_exe { case $PLATFORM in hercules) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/hercules/build_cache} ;; orion) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/orion/build_cache} ;; discover16) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu16/build_cache} ;; discover17) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu17/build_cache} ;; derecho) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) - COMPILERS="intel@2022" + COMPILERS=${COMPILERS:-"intel@2022"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { set +e @@ -66,31 +64,32 @@ case $PLATFORM in TEST_UFSWM=ON ;; gaea) - COMPILERS="intel" + COMPILERS=${COMPILERS:-"intel"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/ncrc/proj/epic/spack-stack/build_cache} ;; hera) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/scratch1/NCEPDEV/nems/role.epic/spack-stack/build_cache} ;; jet) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/build_cache} ;; narwhal) - COMPILERS="intel gcc" + COMPILERS=${COMPILERS:-"intel gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} ;; nautilus) - COMPILERS="intel" + COMPILERS=${COMPILERS:-"intel"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} ;; s4) - COMPILERS="intel" + COMPILERS=${COMPILERS:-"intel"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/data/prod/jedi/spack-stack/build_cache} ;; linux.default) - COMPILERS="gcc" + COMPILERS=${COMPILERS:-"gcc"} + esac echo "Build cache target directory: ${BUILD_CACHE_DIR?'BUILD_CACHE_DIR must be set!'}" From 41bda42cbf699c90ebd086446f7583eec3419b10 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 7 Aug 2024 13:37:50 -0700 Subject: [PATCH 030/134] alert_cmd: show which script failed when using SpackStackBuildCache_AllSteps.sh --- util/weekly_build/ShellSetup.sh | 4 +-- .../SpackStackBuildCache_AllSteps.sh | 26 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index f5ea86a26..fd1ca2af8 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -14,7 +14,7 @@ fi PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} function alert_cmd { - echo "This is a placeholder alerting function. 'alert_cmd' should be defined for each system." + echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." } function spack_install_exe { @@ -59,7 +59,7 @@ case $PLATFORM in PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" INSTALL_OPTS="-j6" function alert_cmd { - mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed. Run ID: $RUNID") + mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") } TEST_UFSWM=ON ;; diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index b7252a71e..dbc9a8d46 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -5,17 +5,19 @@ set -exa cd $(dirname $0) . ShellSetup.sh $* - -trap "ERROR" ERR - -function ERROR { alert_cmd ; exit 1;} - export SETUPDONE=YES -./1_DirectorySetup.sh $* -./2_GetSpackStackDevelop.sh $* -./3_SetupUnifiedEnv.sh $* -./4_SpackInstall.sh $* -./5_BuildCache.sh $* -./6_AppTests.sh $* -./7_Cleanup.sh $* +function trap_and_run { + trap ERROR ERR + scriptname=$1 + function ERROR { alert_cmd $scriptname ; exit 1;} + $* +} + +trap_and_run ./1_DirectorySetup.sh $* +trap_and_run ./2_GetSpackStackDevelop.sh $* +trap_and_run ./3_SetupUnifiedEnv.sh $* +trap_and_run ./4_SpackInstall.sh $* +trap_and_run ./5_BuildCache.sh $* +trap_and_run ./6_AppTests.sh $* +trap_and_run ./7_Cleanup.sh $* From 98f20d7c2017a548e1e7f509e5e93afaa6b1b7eb Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 7 Aug 2024 13:38:12 -0700 Subject: [PATCH 031/134] 7_Cleanup.sh: use set -x --- util/weekly_build/7_Cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/7_Cleanup.sh index 7894356be..150e4ef19 100755 --- a/util/weekly_build/7_Cleanup.sh +++ b/util/weekly_build/7_Cleanup.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi From c8b7393af53bf71c9bb281f09bcf5da536b5097e Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 7 Aug 2024 13:50:30 -0700 Subject: [PATCH 032/134] 1_DirectorySetup.sh: use ShellSetup.sh like everything else --- util/weekly_build/1_DirectorySetup.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/1_DirectorySetup.sh index 726d6e731..f7a2af74a 100755 --- a/util/weekly_build/1_DirectorySetup.sh +++ b/util/weekly_build/1_DirectorySetup.sh @@ -1,7 +1,9 @@ #!/bin/bash +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + set -ex -echo Base directory: ${2?"Second arg: base directory for build"} +echo Base directory: ${RUNDIR:?} -mkdir -p $2 +mkdir -p $RUNDIR From f3727055423865e5f76243e2a4759e9ef1ef66fc Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 7 Aug 2024 13:50:47 -0700 Subject: [PATCH 033/134] add install path padding --- util/weekly_build/3_SetupUnifiedEnv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index bfe08abb4..2f5347cfc 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -15,6 +15,7 @@ for compiler in $COMPILERS; do spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . + spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-256}" # Check for duplicates and fail before doing the "real" concretization with test deps: spack concretize --fresh 2>&1 | tee log.concretize ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf From 39031695a84e7670560bdef1a9659261b28cc0f9 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 10 Sep 2024 06:42:55 -0700 Subject: [PATCH 034/134] Acorn updates for 1.8.0 (#1285) This PR provides updates for the Acorn site config for the 1.8.0 release, which should eventually get merged into develop. --- configs/sites/tier1/acorn/compilers.yaml | 22 ++++++++++--------- configs/sites/tier1/acorn/packages.yaml | 6 +---- configs/sites/tier1/acorn/packages_intel.yaml | 12 ++++++++-- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index e118f32c3..179fcbdbf 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -6,18 +6,17 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel/19.1.3.304 - libfabric environment: - prepend_path: - PATH: /opt/cray/pe/gcc/10.2.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/10.2.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/10.2.0/snos/include set: # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so # Automake-based builds are installed in lib64 @@ -31,11 +30,14 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel-classic/2022.2.0.262 - libfabric environment: diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index fcd79f254..6bd55d67c 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -9,11 +9,6 @@ externals: - spec: git-lfs@2.11.0 modules: [git-lfs/2.11.0] - perl: - buildable: false - externals: - - spec: perl@5.26.1~cpanm+shared+threads - prefix: /usr mysql: buildable: false externals: @@ -38,6 +33,7 @@ gdal: variants: ~curl flex: + buildable: false externals: - spec: flex@2.6.4+lex prefix: /usr diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 34065c870..b9cfc096c 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -31,7 +31,10 @@ - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '^[virtuals=lapack,blas] openblas' + - '@:1.25' + - any_of: ['@:1.24'] + when: '%intel@19.1.3.304' # *DH eckit: require: @@ -39,13 +42,18 @@ when: "%intel@19.1.3.304" eccodes: require: - - any_of: ["@2.27.0"] + - any_of: ["@2.25.0"] when: "%intel@19.1.3.304" + message: "2.25.0 is the last version to use C++11 (as opposed to C++17)" py-scipy: require: - any_of: ["@1.10.1"] when: "%intel@19.1.3.304" cdo: require: + - any_of: ["@2.3.0"] + when: "%intel@2022.0.2.262" + message: "2.3.0 is the last version to use C++17" - any_of: ["@2.0.5"] when: "%intel@19.1.3.304" + message: "2.0.5 is the last version to use C++14" From 58d0c830bbf4899a5e6aae45176ae302a766c0fb Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 10 Sep 2024 20:06:31 +0000 Subject: [PATCH 035/134] add acorn/mirrors.yaml --- configs/sites/tier1/acorn/mirrors.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 configs/sites/tier1/acorn/mirrors.yaml diff --git a/configs/sites/tier1/acorn/mirrors.yaml b/configs/sites/tier1/acorn/mirrors.yaml new file mode 100644 index 000000000..82239d643 --- /dev/null +++ b/configs/sites/tier1/acorn/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null From fbea7f971f4086195463eb3f72a61a37ae8bfe54 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 10 Sep 2024 20:07:17 +0000 Subject: [PATCH 036/134] add log files and limited fetching to 4_SpackInstall.sh --- util/weekly_build/4_SpackInstall.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index d550421ef..77b90235b 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -15,9 +15,13 @@ INSTALL_OPTS="--show-log-on-error --fail-fast $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack fetch + if [ -z $PACKAGES_TO_INSTALL ]; then + spack fetch 2>&1 | tee log.fetch + else + spack fetch --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch + fi # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.test.install # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install done From ecb6b8394d6d37ba236a07327c40c39db73d00f9 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 10 Sep 2024 20:09:38 +0000 Subject: [PATCH 037/134] SpackStackBuildCache_AllSteps.sh: print hostname --- util/weekly_build/SpackStackBuildCache_AllSteps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index dbc9a8d46..8f6e2cb0a 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -1,5 +1,7 @@ #!/bin/bash +echo "build host: $(hostname)" + set -exa cd $(dirname $0) From 2db397951ef0d453ce6d15b85ca608bc6afa8aa1 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 16 Sep 2024 18:37:33 +0000 Subject: [PATCH 038/134] use centralized source cache on acorn --- configs/sites/tier1/acorn/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/sites/tier1/acorn/config.yaml b/configs/sites/tier1/acorn/config.yaml index 56e793047..9f64f0688 100644 --- a/configs/sites/tier1/acorn/config.yaml +++ b/configs/sites/tier1/acorn/config.yaml @@ -1,3 +1,4 @@ config: build_jobs: 6 build_stage: $tempdir/$user/spack-stage + source_cache: /lfs/h1/emc/nceplibs/noscrub/spack-stack/source_cache From e58fcc1ab832cab65efd1dba7f55534d96ce015f Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 16 Sep 2024 19:49:52 +0000 Subject: [PATCH 039/134] use release/1.8.0 for spack submodule (various fixes) --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index a69f3e4f1..5c7383f2a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = spack-stack-dev + branch = release/1.8.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules From 8da2d5e74ba9e1e736310a56ac6c6697ed7ab191 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 16 Sep 2024 21:10:59 +0000 Subject: [PATCH 040/134] update submodule itself for release/1.8.0 --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 33eea0479..1b3e3e525 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 33eea04798da9317b9e48862291f3dfa9788d0e7 +Subproject commit 1b3e3e5252e5b0e4cb28f9e4255e84ea1a06ff5f From 0f3de3292754d6e7d7a4a0dccacf91c3c7522e14 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 17 Sep 2024 01:35:49 +0000 Subject: [PATCH 041/134] update fetch logic --- util/weekly_build/4_SpackInstall.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 77b90235b..27bd373e4 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -15,10 +15,10 @@ INSTALL_OPTS="--show-log-on-error --fail-fast $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - if [ -z $PACKAGES_TO_INSTALL ]; then - spack fetch 2>&1 | tee log.fetch + if [ -z "$PACKAGES_TO_INSTALL" ]; then + spack fetch --missing 2>&1 | tee log.fetch else - spack fetch --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch + spack fetch --missing --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch fi # Just install the packages we're testing (+dependencies): spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.test.install From c46e345fbdda01e55b6408a83a6f1c64612719bd Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 17 Sep 2024 01:36:52 +0000 Subject: [PATCH 042/134] continued updates for acorn config --- util/weekly_build/ShellSetup.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index fd1ca2af8..c73266999 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -46,14 +46,19 @@ case $PLATFORM in COMPILERS=${COMPILERS:-"intel@2022"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { - set +e - ( qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & - ( qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & - wait - rc=$? - set -e - cat spack-build-cache-${RUNID}* - return $rc +# set +e +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# wait +# rc=$? +# set -e +# cat spack-build-cache-${RUNID}* +# return $rc +## cp ${SPACK_STACK_DIR:?}/util/acorn/{build.pbs,spackinstall.sh} ${SPACK_ENV}/. +## /opt/pbs/bin/qsub -Wblock=true ${SPACK_ENV}/build.pbs +## spack $* | tee -a log.install 2>&1 + shift 1 + ${SPACK_STACK_DIR}/util/parallel_install.sh 2 4 $* } PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" From 9f421bcb4a38045af14f50bb8243fd18ba56991e Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 17 Sep 2024 21:01:42 +0000 Subject: [PATCH 043/134] more acorn updates --- util/weekly_build/ShellSetup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index c73266999..9d0cca5bf 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -43,6 +43,8 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) + # Python module needed for Spack backend to avoid resource misplacement bug: + module load gcc/10.3.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { From 65bd971f35d4335cd69d8d0e52d3520fa1f4cd99 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 17 Sep 2024 21:02:04 +0000 Subject: [PATCH 044/134] set padded_length:200 instead of 256 (hdf5/cmake failure) --- util/weekly_build/3_SetupUnifiedEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/3_SetupUnifiedEnv.sh index 2f5347cfc..3479ed7ec 100755 --- a/util/weekly_build/3_SetupUnifiedEnv.sh +++ b/util/weekly_build/3_SetupUnifiedEnv.sh @@ -15,7 +15,7 @@ for compiler in $COMPILERS; do spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-256}" + spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization with test deps: spack concretize --fresh 2>&1 | tee log.concretize ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf From 14d86b98dea3a74bb3096e2e7e43845f7c7b11cb Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 18 Sep 2024 00:19:57 +0000 Subject: [PATCH 045/134] always use --no-cache so cache is always rebuilt --- util/weekly_build/4_SpackInstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 27bd373e4..7c98a2c6b 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -10,7 +10,7 @@ set +x . setup.sh set -x -INSTALL_OPTS="--show-log-on-error --fail-fast $INSTALL_OPTS" +INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} From faf2691c5edbedbef08412b924882a1378975e72 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 18 Sep 2024 00:20:26 +0000 Subject: [PATCH 046/134] more acorn config --- util/weekly_build/ShellSetup.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 9d0cca5bf..427a73ba8 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -43,9 +43,8 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} ;; acorn) - # Python module needed for Spack backend to avoid resource misplacement bug: - module load gcc/10.3.0 python/3.11.7 - COMPILERS=${COMPILERS:-"intel@2022"} + module load gcc/11.2.0 python/3.11.7 + COMPILERS=${COMPILERS:-"intel@2022.2.0.262 intel@19.1.3.304"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { # set +e @@ -60,12 +59,12 @@ case $PLATFORM in ## /opt/pbs/bin/qsub -Wblock=true ${SPACK_ENV}/build.pbs ## spack $* | tee -a log.install 2>&1 shift 1 - ${SPACK_STACK_DIR}/util/parallel_install.sh 2 4 $* + ${SPACK_STACK_DIR}/util/parallel_install.sh 3 4 $* } PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" - PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env upp-env" - INSTALL_OPTS="-j6" + PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" function alert_cmd { + module purge # annoying libstdc++ issue mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") } TEST_UFSWM=ON From dcd4f1b8a8dc8e893de5c6d4ff99b89e85e53406 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 19 Sep 2024 02:48:16 +0000 Subject: [PATCH 047/134] add option to keep weekly build directory --- util/weekly_build/7_Cleanup.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/7_Cleanup.sh index 150e4ef19..2d7a1f84e 100755 --- a/util/weekly_build/7_Cleanup.sh +++ b/util/weekly_build/7_Cleanup.sh @@ -4,4 +4,6 @@ set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi -/usr/bin/rm -rf ${RUNDIR:?}/${RUNID:?} +if [ "$KEEP_WEEKLY_BUILD_DIR" != YES ]; then + /usr/bin/rm -rf ${RUNDIR:?}/${RUNID:?} +fi From 2e578e42129a8442f1274dedee473252cdc734de Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 19 Sep 2024 02:48:48 +0000 Subject: [PATCH 048/134] ShellSetup.sh: fix typo in acorn compilers --- util/weekly_build/ShellSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 427a73ba8..2700bdd4e 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -44,7 +44,7 @@ case $PLATFORM in ;; acorn) module load gcc/11.2.0 python/3.11.7 - COMPILERS=${COMPILERS:-"intel@2022.2.0.262 intel@19.1.3.304"} + COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { # set +e From 6535e77c4630910a71269b710b0481b45988f1ee Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 19 Sep 2024 02:49:36 +0000 Subject: [PATCH 049/134] chmod +x apptests/test_ufswm.sh --- util/weekly_build/apptests/test_ufswm.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 util/weekly_build/apptests/test_ufswm.sh diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh old mode 100644 new mode 100755 From 44d4e7cea5f19aeecfb3b693c7d294856e559758 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Sep 2024 16:02:59 -0600 Subject: [PATCH 050/134] Update util/weekly_build/ShellSetup.sh for Nautilus and add first step of nightly builds Nautilus using GitHub actions --- .../Nightly_01_DirectorySetup/action.yaml | 14 ++++++++ .github/workflows/nautilus-nightly.yaml | 32 +++++++++++++++++++ util/weekly_build/ShellSetup.sh | 4 +-- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .github/actions/Nightly_01_DirectorySetup/action.yaml create mode 100644 .github/workflows/nautilus-nightly.yaml diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml new file mode 100644 index 000000000..e62caa2d8 --- /dev/null +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -0,0 +1,14 @@ +name: Nightly 01 Directory Setup +#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller + +runs: + using: "composite" + steps: + - name: Nightly 01 Directory Setup + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + RUNDIR=${BASE_DIR}/${RUNID} + [ -d {RUNDIR} ] && rm -fr ${RUNDIR} + cd util/weekly_build + ./1_DirectorySetup.sh ${RUNID} ${RUNDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml new file mode 100644 index 000000000..08341e039 --- /dev/null +++ b/.github/workflows/nautilus-nightly.yaml @@ -0,0 +1,32 @@ +name: nautilus heinzell auto ci + +on: + workflow_dispatch: + pull_request: + branches: + - develop + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + BASE_DIR: /p/work2/heinzell/spack-stack-nightly + PLATFORM: nautilus + +jobs: + + nightly-build: + runs-on: ['self-hosted','nautilus-heinzell'] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + clean: true + - name: 01_DirectorySetup + uses: ./.github/actions/Nightly_01_DirectorySetup diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 2700bdd4e..46ad0ef5a 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -86,8 +86,8 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} ;; nautilus) - COMPILERS=${COMPILERS:-"intel"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} + COMPILERS=${COMPILERS:-"intel oneapi gcc"} + BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache} ;; s4) COMPILERS=${COMPILERS:-"intel"} From 544f5a73e8a3bde216366b0ca7652bfdc6e49bbf Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Sep 2024 16:12:15 -0600 Subject: [PATCH 051/134] Add step 2 for nightly builds on Nautilus: GetSpackStackDevelop --- .../actions/Nightly_01_DirectorySetup/action.yaml | 8 +++++--- .../Nightly_02_GetSpackStackDevelop/action.yaml | 12 ++++++++++++ .github/workflows/nautilus-nightly.yaml | 4 +++- util/weekly_build/2_GetSpackStackDevelop.sh | 3 ++- 4 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 .github/actions/Nightly_02_GetSpackStackDevelop/action.yaml diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index e62caa2d8..d1ebb76c3 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -8,7 +8,9 @@ runs: shell: bash run: | RUNID=`date +"%Y%m%d"` - RUNDIR=${BASE_DIR}/${RUNID} - [ -d {RUNDIR} ] && rm -fr ${RUNDIR} + # DH* TODO REMOVE LATER? + RUNDIR=${BASEDIR}/${RUNID} + rm -vfr ${RUNDIR} + # *DH cd util/weekly_build - ./1_DirectorySetup.sh ${RUNID} ${RUNDIR} ${PLATFORM} + ./1_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml new file mode 100644 index 000000000..cb6141781 --- /dev/null +++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml @@ -0,0 +1,12 @@ +name: Nightly 02 Get Spack-Stack Develop +#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller + +runs: + using: "composite" + steps: + - name: Nightly 02 Get Spack-Stack Develop + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./2_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml index 08341e039..07bdcce09 100644 --- a/.github/workflows/nautilus-nightly.yaml +++ b/.github/workflows/nautilus-nightly.yaml @@ -16,7 +16,7 @@ defaults: shell: bash env: - BASE_DIR: /p/work2/heinzell/spack-stack-nightly + BASEDIR: /p/work2/heinzell/spack-stack-nightly PLATFORM: nautilus jobs: @@ -30,3 +30,5 @@ jobs: clean: true - name: 01_DirectorySetup uses: ./.github/actions/Nightly_01_DirectorySetup + - name: 02_GetSpackStackDevelop + uses: ./.github/actions/Nightly_02_GetSpackStackDevelop \ No newline at end of file diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index 67e95b136..fc61ffa46 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -9,5 +9,6 @@ if [ -d $RUNID ]; then cd $RUNID git pull else - git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID + #git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID + git clone --recurse-submodules https://github.nrlmry.navy.mil/JCSDA/spack-stack -b feature/weekly_build_nautilus $RUNID fi From e6e0b772be1c598e2ae2a3030f3cf6ac536bd1b5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Sep 2024 16:26:52 -0600 Subject: [PATCH 052/134] Revert me: remove other CI workflows on self-hosted runners --- .github/workflows/macos-ci-aarch64.yaml | 178 ------------- .../ubuntu-ci-containers-x86_64.yaml | 111 -------- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 209 --------------- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 241 ----------------- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 249 ------------------ 5 files changed, 988 deletions(-) delete mode 100644 .github/workflows/macos-ci-aarch64.yaml delete mode 100644 .github/workflows/ubuntu-ci-containers-x86_64.yaml delete mode 100644 .github/workflows/ubuntu-ci-x86_64-gnu.yaml delete mode 100644 .github/workflows/ubuntu-ci-x86_64-intel.yaml delete mode 100644 .github/workflows/ubuntu-ci-x86_64-oneapi.yaml diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml deleted file mode 100644 index 3087b7f1f..000000000 --- a/.github/workflows/macos-ci-aarch64.yaml +++ /dev/null @@ -1,178 +0,0 @@ -name: macos-ci-aarch64-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - macos-ci-aarch64-build: - runs-on: [macos-ci-aarch64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - find ./* -type d -exec chmod u+xw {} \; - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /Users/ec2-user/spack-stack/build-cache - mkdir -p /Users/ec2-user/spack-stack/source-cache - - - name: create-env - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up homebrew and lmod support - eval "$(/opt/homebrew/bin/brew shellenv)" - source /opt/homebrew/opt/lmod/init/profile - - # Set up spack-stack - source ./setup.sh - # Important! - export SPACK_PYTHON=/usr/bin/python3 - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - - spack clean -ab - spack bootstrap now - - spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang - spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python - spack external find --scope system perl - spack external find --scope system libiconv - spack external find --scope system wget - PATH="/opt/homebrew/opt/curl/bin:$PATH" \ - spack external find --scope system curl - PATH="/opt/homebrew/opt/qt5/bin:$PATH" \ - spack external find --scope system qt - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - export -n SPACK_SYSTEM_CONFIG_PATH - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" - spack config add "packages:all:compiler:[apple-clang@14.0.3]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - # DH* 20240513 - avoid hdf-eos2 until https://github.com/spack/spack/issues/44168 is resolved - #spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - spack config add "packages:met:variants:+python +grib2 +graphics" - # *DH - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ - spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Create modules - spack clean -a - spack module lmod refresh -y - spack stack setup-meta-modules - - # Next steps: synchronize source and build cache to a central/combined mirror? - spack env deactivate - - # Test environment chaining - echo "Test environment chaining" - spack stack create env --name chaintest --template empty --site macos.default --upstream ${ENVDIR}/install --compiler apple-clang - # Retain config from upstream so we don't have to rebuild: - cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - spack env activate ${PWD}/envs/chaintest - spack add nccmp@1.9.0.1%apple-clang - spack concretize | tee envs/chaintest/log.concretize - unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - spack env deactivate - - - name: test-env - run: | - # Set up homebrew and lmod support - eval "$(/opt/homebrew/bin/brew shellenv)" - source /opt/homebrew/opt/lmod/init/profile - - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-apple-clang/14.0.3 - module load stack-openmpi/5.0.3 - module load stack-python/3.11.7 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml deleted file mode 100644 index b9265c094..000000000 --- a/.github/workflows/ubuntu-ci-containers-x86_64.yaml +++ /dev/null @@ -1,111 +0,0 @@ -name: ubuntu-ci-container-x86_64-build -on: - # Uncomment this to test for PRs (but do not submit) - #pull_request: - # paths-ignore: - # - 'configs/sites/**' - # - 'doc/**' - # - '**.md' - # - '.github/ISSUE_TEMPLATE/*' - # - '.gitignore' - # - schedule: - - cron: '0 8 * * *' - workflow_dispatch: - inputs: - container: - description: 'Container template (input requred, no default value).' - required: true - specs: - description: 'Which specs to add to the template (input requred, no default value).' - required: true - -defaults: - run: - shell: bash - -jobs: - ubuntu-ci-container-x86_64-build: - runs-on: [ubuntu-ci-c6a-x86_64] - - steps: - - name: checkout - uses: actions/checkout@v3 - with: - submodules: true - - # Ensure that ubuntu can run docker - - name: ubuntu-docker - run: | - echo "Checking if user ubuntu can run docker" - if id -nG ubuntu | grep -qw docker; then - echo "User ubuntu already belongs to group docker" - else - echo "Adding user ubuntu to group docker" - sudo gpasswd -a ubuntu docker - newgrp docker - docker run hello-world - fi - - - name: create-ctr - run: | - source ./setup.sh - - # Get day of week to set default container for scheduled builds - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 1 || $DOW == 4 ]]; then - export CONTAINER=${{ inputs.container || 'docker-ubuntu-clang-mpich' }} - export SPECS=${{ inputs.specs || 'jedi-ci' }} - elif [[ $DOW == 2 || $DOW == 5 ]]; then - export CONTAINER=${{ inputs.container || 'docker-ubuntu-gcc-openmpi' }} - export SPECS=${{ inputs.specs || 'jedi-ci' }} - elif [[ $DOW == 3 || $DOW == 6 ]]; then - export CONTAINER=${{ inputs.container || 'docker-ubuntu-intel-impi' }} - export SPECS=${{ inputs.specs || 'jedi-ci' }} - else - # Day 7: The Sabbath of rest (but do some house keeping later on) - exit 0 - fi - - export ENVDIR=$PWD/envs/${CONTAINER} - spack stack create ctr --container ${CONTAINER} --specs ${SPECS} - - cd ${ENVDIR} - spack containerize > Dockerfile - docker build -t ${CONTAINER}-${SPECS} . - - # Report status to JCSDA CI slack channel for nightly runs only - - name: Report Status - if: always() - uses: ravsamhq/notify-slack-action@v1 - env: - SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} - with: - # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url - # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script - # https://github.com/marketplace/actions/notify-slack-action - status: ${{ job.status }} - notify_when: 'success,failure,warnings' - notification_title: '{workflow} has {status_message}' - message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>' - footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }} - # For testing: only notify user Dom - #mention_users: 'U02NLGXF5HV' - #mention_users_when: 'failure,warnings' - # Default: notify channel - mention_groups: '!channel' - mention_groups_when: 'failure,warnings' - - # We can only clean the docker image registry *after* reporting - # the status to the CI slack channel, because the cleanup is - # deleting the docker image for ravsamhq/notify-slack-action@v1 - - name: clean-docker-registry - if: always() - run: | - # Get day of week (only run this on Sunday, which is 7) - DOW=$(date +%u) - if [[ $DOW == 7 ]]; then - echo "Pruning all docker images" - docker system prune -a -f - fi diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml deleted file mode 100644 index 13414615c..000000000 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ /dev/null @@ -1,209 +0,0 @@ -name: ubuntu-ci-c6a-x86_64-gnu-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - ubuntu-ci-c6a-x86_64-gnu-build: - runs-on: [ubuntu-ci-c6a-x86_64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - find ./* -type d -exec chmod u+xw {} \; - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ - - - name: create-buildcache - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up spack-stack - source ./setup.sh - - declare -a TEMPLATES=("unified-dev" "skylab-dev") - for TEMPLATE in "${TEMPLATES[@]}"; do - if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then - export ENVNAME=ue-gcc-11.4.0-buildcache - elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then - export ENVNAME=se-gcc-11.4.0-buildcache - fi - echo "Creating environment ${ENVNAME} from template ${TEMPLATE}" - - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site linux.default --template ${TEMPLATE} --name ${ENVNAME} --compiler gcc - spack env activate ${ENVDIR} - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python - spack external find --scope system sed - spack external find --scope system perl - spack external find --scope system wget - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - export -n SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" - spack config add "packages:all:compiler:[gcc@11.4.0]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.gnu-11.4.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0-buildcache -i fms -i crtm -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env - - # the rest - echo "${TEMPLATE} ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate - - done - - - name: create-env - run: | - source ./setup.sh - export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache - export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-gcc-11.4.0 - export ENVDIR=$PWD/envs/${ENVNAME} - rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ - spack env activate ${ENVDIR} - - # Concretize - spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl - - # Add binary cache back in and reindex it - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - echo "Packages in combined spack build caches:" - spack buildcache list - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env - - ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check - spack clean -a - spack module tcl refresh -y - spack stack setup-meta-modules - spack env deactivate - - # Test environment chaining - echo "Test environment chaining" - spack stack create env --name chaintest --template empty --site linux.default --compiler gcc --upstream ${ENVDIR}/install - # Retain config from upstream so we don't have to rebuild: - cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - spack env activate ${PWD}/envs/chaintest - spack add nccmp@1.9.0.1%gcc - spack concretize | tee envs/chaintest/log.concretize - unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - spack env deactivate - echo "Verify 'create env' warnings" - echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/site/packages.yaml - echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml - spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt - cnt=$(grep -c "WARNING.*do not match" stderr.txt || true) - if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi - - - name: test-env - run: | - source /etc/profile.d/modules.sh - module use /home/ubuntu/spack-stack/modulefiles - - export ENVNAME=ue-gcc-11.4.0 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-gcc/11.4.0 - module load stack-openmpi/5.0.3 - module load stack-python/3.11.7 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml deleted file mode 100644 index d1ed33d65..000000000 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ /dev/null @@ -1,241 +0,0 @@ -name: ubuntu-ci-c6a-x86_64-intel-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - ubuntu-ci-c6a-x86_64-intel-build: - runs-on: [ubuntu-ci-c6a-x86_64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - find ./* -type d -exec chmod u+xw {} \; - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ - - - name: create-buildcache - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up spack-stack - source ./setup.sh - export ENVNAME=ue-intel-2021.10.0-buildcache - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler intel - spack env activate ${ENVDIR} - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python - spack external find --scope system sed - spack external find --scope system perl - spack external find --scope system wget - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - # For Intel - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " spec: intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cxx: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/icpc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " f77: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " fc: /opt/intel/oneapi/compiler/2023.2.3/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " environment:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " prepend_path:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2023.2.3/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - - # Need to find external Intel MPI and annotate with the - # correct compiler, no way to do that with spack commands. - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - # Add external Intel MKL and oneAPI runtime - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - #echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - #echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - #echo " - spec: intel-oneapi-runtime@2023.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - #echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - # Add external qt@5 to build ecflow - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " qt::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " buildable: False" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: qt@5.15.3" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /usr" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - export -n SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]" - spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Don't generate qt module when using external package - spack config add "modules:default:tcl:exclude:[qt]" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.intel-2021.10.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0-buildcache -i fms -i crtm -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate - - - name: create-env - run: | - source ./setup.sh - export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache - export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-intel-2021.10.0 - export ENVDIR=$PWD/envs/${ENVNAME} - rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ - spack env activate ${ENVDIR} - - # Concretize - spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl - - # Add binary cache back in and reindex it - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - echo "Packages in combined spack build caches:" - spack buildcache list - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env - - ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check - spack clean -a - spack module tcl refresh -y - spack stack setup-meta-modules - spack env deactivate - - # Test environment chaining - echo "Test environment chaining" - spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel - # Retain config from upstream so we don't have to rebuild: - cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - spack env activate ${PWD}/envs/chaintest - spack add nccmp@1.9.0.1%intel - spack concretize | tee envs/chaintest/log.concretize - unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - spack env deactivate - - - name: test-env - run: | - source /etc/profile.d/modules.sh - module use /home/ubuntu/spack-stack/modulefiles - - export ENVNAME=ue-intel-2021.10.0 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-intel-oneapi-mpi/2021.10.0 - module load stack-python/3.11.7 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml deleted file mode 100644 index e2bf9e604..000000000 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ /dev/null @@ -1,249 +0,0 @@ -name: ubuntu-ci-c6a-x86_64-oneapi-build -on: - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - ubuntu-ci-c6a-x86_64-oneapi-build: - runs-on: [ubuntu-ci-c6a-x86_64] - - steps: - - name: cleanup - run: | - pwd - ls -lart - find ./* -type d -exec chmod u+xw {} \; - rm -fr * - - - name: checkout - uses: actions/checkout@v4 - with: - submodules: true - - - name: prepare-directories - run: | - mkdir -p /home/ubuntu/spack-stack/build-cache/ - mkdir -p /home/ubuntu/spack-stack/source-cache/ - - - name: create-buildcache - run: | - # Get day of week to decide whether to use build caches or not - DOW=$(date +%u) - # Monday is 1 ... Sunday is 7 - if [[ $DOW == 7 ]]; then - export USE_BINARY_CACHE=false - echo "Ignore existing binary cache for creating buildcache environment" - else - export USE_BINARY_CACHE=true - echo "Use existing binary cache for creating buildcache environment" - fi - - # Set up spack-stack - source ./setup.sh - export ENVNAME=ue-oneapi-2024.2.0-buildcache - export ENVDIR=$PWD/envs/${ENVNAME} - spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi - spack env activate ${ENVDIR} - export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" - - # Find external packages - spack external find --scope system \ - --exclude bison --exclude openssl \ - --exclude curl --exclude python \ - --exclude gmake - spack external find --scope system sed - spack external find --scope system perl - spack external find --scope system wget - spack external find --scope system texlive - spack external find --scope system mysql - - # Find compilers - spack compiler find --scope system - - # For Intel oneAPI - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " f77: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " fc: /opt/intel/oneapi/compiler/2024.2/bin/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml - - # Need to find external Intel MPI and annotate with the - # correct compiler, no way to do that with spack commands. - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - # Add external Intel MKL and oneAPI runtime - echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - - export -n SPACK_SYSTEM_CONFIG_PATH - - # For buildcaches - spack config add config:install_tree:padded_length:200 - - # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" - spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" - - # Add additional variants for MET packages, different from config/common/packages.yaml - spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - - # Switch providers for blas, lapack, fftw-api - spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]" - spack config add "packages:all:providers:blas:[intel-oneapi-mkl]" - spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]" - spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]" - spack config add "packages:ectrans:require:'+mkl ~fftw'" - spack config add "packages:gsibec:require:'+mkl'" - spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']" - - # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - spack config add "packages:gmake:require:'@:4.2'" - - # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI - sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` - - # Don't generate ecflow module when using external package - spack config add "modules:default:tcl:exclude:[ecflow]" - - # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.oneapi-2024.2.0-buildcache - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl - - # Add and update source cache - spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ - spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ - - # Add binary cache if requested - if [ "$USE_BINARY_CACHE" = true ] ; then - set +e - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) - set +e - echo "Packages in spack binary cache:" - spack buildcache list - fi - - # Break installation up in pieces and create build caches in between - # This allows us to "spin up" builds that altogether take longer than - # six hours, and/or fail later in the build process. - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env - - # jedi-base-env - echo "jedi-base-env ..." - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env - - # the rest - echo "unified-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ - - # Remove binary cache for next round of concretization - if [ "$USE_BINARY_CACHE" = true ] ; then - spack mirror rm local-binary - fi - - # Remove buildcache config settings - spack config remove config:install_tree:padded_length - - # Next steps: synchronize source and build cache to a central/combined mirror? - - # Cleanup - spack clean -a - spack env deactivate - - - name: create-env - run: | - source ./setup.sh - export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache - export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} - export ENVNAME=ue-oneapi-2024.2.0 - export ENVDIR=$PWD/envs/${ENVNAME} - rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ - spack env activate ${ENVDIR} - - # Concretize - spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl - - # Add binary cache back in and reindex it - spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ - echo "Packages in combined spack build caches:" - spack buildcache list - - # base-env - echo "base-env ..." - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env - - ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check - spack clean -a - spack module tcl refresh -y - spack stack setup-meta-modules - spack env deactivate - - ### # Test environment chaining - ### echo "Test environment chaining" - ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel - ### # Retain config from upstream so we don't have to rebuild: - ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - ### spack env activate ${PWD}/envs/chaintest - ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - ### spack config add "packages:gmake:require:'@:4.2'" - ### spack add nccmp@1.9.0.1%oneapi - ### spack concretize | tee envs/chaintest/log.concretize - ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - ### spack env deactivate - - - name: test-env - run: | - source /etc/profile.d/modules.sh - module use /home/ubuntu/spack-stack/modulefiles - - export ENVNAME=ue-oneapi-2024.2.0 - export ENVDIR=$PWD/envs/${ENVNAME} - ls -l ${ENVDIR}/install/modulefiles/Core - - module use ${ENVDIR}/install/modulefiles/Core - module load stack-oneapi/2024.2.0 - module load stack-intel-oneapi-mpi/2021.13 - module load stack-python/3.11.7 - module available - - module load jedi-ufs-env - module load ewok-env - module load soca-env - module list From 3a979065a899bd64e6220d9706af2d6ea9af6800 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Sep 2024 16:34:07 -0600 Subject: [PATCH 053/134] Add .github/actions/Nightly_03_SetupUnifiedEnv --- .github/actions/Nightly_01_DirectorySetup/action.yaml | 1 - .../Nightly_02_GetSpackStackDevelop/action.yaml | 1 - .../actions/Nightly_03_SetupUnifiedEnv/action.yaml | 11 +++++++++++ .github/workflows/nautilus-nightly.yaml | 4 +++- 4 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .github/actions/Nightly_03_SetupUnifiedEnv/action.yaml diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index d1ebb76c3..9773eecc1 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -1,5 +1,4 @@ name: Nightly 01 Directory Setup -#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller runs: using: "composite" diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml index cb6141781..246b17484 100644 --- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml +++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml @@ -1,5 +1,4 @@ name: Nightly 02 Get Spack-Stack Develop -#description: Display a GitHub Actions log files - requires CI_DIR environment variable from caller runs: using: "composite" diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml new file mode 100644 index 000000000..2b4bb64ae --- /dev/null +++ b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 03 Setup Unified Env + +runs: + using: "composite" + steps: + - name: Nightly 03 Setup Unified Env + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./3_SetupUnifiedEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml index 07bdcce09..69ad4a85e 100644 --- a/.github/workflows/nautilus-nightly.yaml +++ b/.github/workflows/nautilus-nightly.yaml @@ -31,4 +31,6 @@ jobs: - name: 01_DirectorySetup uses: ./.github/actions/Nightly_01_DirectorySetup - name: 02_GetSpackStackDevelop - uses: ./.github/actions/Nightly_02_GetSpackStackDevelop \ No newline at end of file + uses: ./.github/actions/Nightly_02_GetSpackStackDevelop + - name: 03_SetupUnifiedEnv + uses: ./.github/actions/Nightly_02_SetupUnifiedEnv From 5ab19610595661a6d248f3549ca6e0da61c94478 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 19 Sep 2024 16:39:43 -0600 Subject: [PATCH 054/134] Add .github/actions/Nightly_04_SpackInstall --- .github/actions/Nightly_04_SpackInstall/action.yaml | 11 +++++++++++ .github/workflows/nautilus-nightly.yaml | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 .github/actions/Nightly_04_SpackInstall/action.yaml diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml new file mode 100644 index 000000000..ea90e5755 --- /dev/null +++ b/.github/actions/Nightly_04_SpackInstall/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 04 Spack Install + +runs: + using: "composite" + steps: + - name: Nightly 04 Spack Install + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./4_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly.yaml index 69ad4a85e..06ebdb6e3 100644 --- a/.github/workflows/nautilus-nightly.yaml +++ b/.github/workflows/nautilus-nightly.yaml @@ -33,4 +33,6 @@ jobs: - name: 02_GetSpackStackDevelop uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - name: 03_SetupUnifiedEnv - uses: ./.github/actions/Nightly_02_SetupUnifiedEnv + uses: ./.github/actions/Nightly_03_SetupUnifiedEnv + - name: 04_SpackInstall + uses: ./.github/actions/Nightly_04_SpackInstall From 3fa47c5795928cbd60b9e4fc9686d4ee922c9755 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 20 Sep 2024 05:52:55 -0600 Subject: [PATCH 055/134] Update .github/* for nautilus nightly oneapi --- .../actions/Nightly_01_DirectorySetup/action.yaml | 2 +- .../Nightly_02_GetSpackStackDevelop/action.yaml | 2 +- .../actions/Nightly_03_SetupUnifiedEnv/action.yaml | 2 +- .../actions/Nightly_04_SpackInstall/action.yaml | 2 +- ...s-nightly.yaml => nautilus-nightly-oneapi.yaml} | 5 ++++- .../{1_DirectorySetup.sh => 01_DirectorySetup.sh} | 0 ...kStackDevelop.sh => 02_GetSpackStackDevelop.sh} | 0 .../{3_SetupUnifiedEnv.sh => 03_SetupEnv.sh} | 0 .../{4_SpackInstall.sh => 04_SpackInstall.sh} | 0 .../{5_BuildCache.sh => 05_BuildCache.sh} | 0 .../weekly_build/{6_AppTests.sh => 06_AppTests.sh} | 0 util/weekly_build/{7_Cleanup.sh => 07_Cleanup.sh} | 0 util/weekly_build/ShellSetup.sh | 2 ++ util/weekly_build/SpackStackBuildCache_AllSteps.sh | 14 +++++++------- 14 files changed, 17 insertions(+), 12 deletions(-) rename .github/workflows/{nautilus-nightly.yaml => nautilus-nightly-oneapi.yaml} (90%) rename util/weekly_build/{1_DirectorySetup.sh => 01_DirectorySetup.sh} (100%) rename util/weekly_build/{2_GetSpackStackDevelop.sh => 02_GetSpackStackDevelop.sh} (100%) rename util/weekly_build/{3_SetupUnifiedEnv.sh => 03_SetupEnv.sh} (100%) rename util/weekly_build/{4_SpackInstall.sh => 04_SpackInstall.sh} (100%) rename util/weekly_build/{5_BuildCache.sh => 05_BuildCache.sh} (100%) rename util/weekly_build/{6_AppTests.sh => 06_AppTests.sh} (100%) rename util/weekly_build/{7_Cleanup.sh => 07_Cleanup.sh} (100%) diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index 9773eecc1..e58d305ca 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -12,4 +12,4 @@ runs: rm -vfr ${RUNDIR} # *DH cd util/weekly_build - ./1_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} + ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml index 246b17484..7565046dd 100644 --- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml +++ b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml @@ -8,4 +8,4 @@ runs: run: | RUNID=`date +"%Y%m%d"` cd util/weekly_build - ./2_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM} + ./02_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml index 2b4bb64ae..0f598cb88 100644 --- a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml +++ b/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml @@ -8,4 +8,4 @@ runs: run: | RUNID=`date +"%Y%m%d"` cd util/weekly_build - ./3_SetupUnifiedEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM} + ./03_SetupEnv.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml index ea90e5755..0954971b1 100644 --- a/.github/actions/Nightly_04_SpackInstall/action.yaml +++ b/.github/actions/Nightly_04_SpackInstall/action.yaml @@ -8,4 +8,4 @@ runs: run: | RUNID=`date +"%Y%m%d"` cd util/weekly_build - ./4_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM} + ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml similarity index 90% rename from .github/workflows/nautilus-nightly.yaml rename to .github/workflows/nautilus-nightly-oneapi.yaml index 06ebdb6e3..5c8b33f9f 100644 --- a/.github/workflows/nautilus-nightly.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -18,11 +18,14 @@ defaults: env: BASEDIR: /p/work2/heinzell/spack-stack-nightly PLATFORM: nautilus + COMPILER: oneapi jobs: - nightly-build: + nightly-build-oneapi: runs-on: ['self-hosted','nautilus-heinzell'] + # Set timeout to 12hrs + #timeout-minutes: 720 steps: - uses: actions/checkout@v4 with: diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/01_DirectorySetup.sh similarity index 100% rename from util/weekly_build/1_DirectorySetup.sh rename to util/weekly_build/01_DirectorySetup.sh diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/02_GetSpackStackDevelop.sh similarity index 100% rename from util/weekly_build/2_GetSpackStackDevelop.sh rename to util/weekly_build/02_GetSpackStackDevelop.sh diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupEnv.sh similarity index 100% rename from util/weekly_build/3_SetupUnifiedEnv.sh rename to util/weekly_build/03_SetupEnv.sh diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh similarity index 100% rename from util/weekly_build/4_SpackInstall.sh rename to util/weekly_build/04_SpackInstall.sh diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/05_BuildCache.sh similarity index 100% rename from util/weekly_build/5_BuildCache.sh rename to util/weekly_build/05_BuildCache.sh diff --git a/util/weekly_build/6_AppTests.sh b/util/weekly_build/06_AppTests.sh similarity index 100% rename from util/weekly_build/6_AppTests.sh rename to util/weekly_build/06_AppTests.sh diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/07_Cleanup.sh similarity index 100% rename from util/weekly_build/7_Cleanup.sh rename to util/weekly_build/07_Cleanup.sh diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 46ad0ef5a..24767ea78 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -86,6 +86,8 @@ case $PLATFORM in BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} ;; nautilus) + module purge + umask 0022 COMPILERS=${COMPILERS:-"intel oneapi gcc"} BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache} ;; diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 8f6e2cb0a..f9b85eae6 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -16,10 +16,10 @@ function trap_and_run { $* } -trap_and_run ./1_DirectorySetup.sh $* -trap_and_run ./2_GetSpackStackDevelop.sh $* -trap_and_run ./3_SetupUnifiedEnv.sh $* -trap_and_run ./4_SpackInstall.sh $* -trap_and_run ./5_BuildCache.sh $* -trap_and_run ./6_AppTests.sh $* -trap_and_run ./7_Cleanup.sh $* +trap_and_run ./01_DirectorySetup.sh $* +trap_and_run ./02_GetSpackStackDevelop.sh $* +trap_and_run ./03_SetupEnv.sh $* +trap_and_run ./04_SpackInstall.sh $* +trap_and_run ./05_BuildCache.sh $* +trap_and_run ./06_AppTests.sh $* +trap_and_run ./07_Cleanup.sh $* From 0599b662191f03a991563e4b462d85d60e8de4ff Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 20 Sep 2024 17:51:13 +0000 Subject: [PATCH 056/134] make spack-stack url & branch configurable --- util/weekly_build/2_GetSpackStackDevelop.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStackDevelop.sh index 67e95b136..368d13522 100755 --- a/util/weekly_build/2_GetSpackStackDevelop.sh +++ b/util/weekly_build/2_GetSpackStackDevelop.sh @@ -9,5 +9,5 @@ if [ -d $RUNID ]; then cd $RUNID git pull else - git clone --recurse-submodules https://github.com/AlexanderRichert-NOAA/spack-stack -b weekly_build $RUNID + git clone --recurse-submodules ${SPACK_STACK_URL:-https://github.com/AlexanderRichert-NOAA/spack-stack} -b ${SPACK_STACK_BRANCH:-weekly_build} $RUNID fi From df0a7c3a59a886dc5d125bceda08676abd3ee9be Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 20 Sep 2024 17:51:59 +0000 Subject: [PATCH 057/134] rename GetSpackStack script --- .../{2_GetSpackStackDevelop.sh => 2_GetSpackStack.sh} | 0 util/weekly_build/SpackStackBuildCache_AllSteps.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename util/weekly_build/{2_GetSpackStackDevelop.sh => 2_GetSpackStack.sh} (100%) diff --git a/util/weekly_build/2_GetSpackStackDevelop.sh b/util/weekly_build/2_GetSpackStack.sh similarity index 100% rename from util/weekly_build/2_GetSpackStackDevelop.sh rename to util/weekly_build/2_GetSpackStack.sh diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 8f6e2cb0a..df04a5e58 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -17,7 +17,7 @@ function trap_and_run { } trap_and_run ./1_DirectorySetup.sh $* -trap_and_run ./2_GetSpackStackDevelop.sh $* +trap_and_run ./2_GetSpackStack.sh $* trap_and_run ./3_SetupUnifiedEnv.sh $* trap_and_run ./4_SpackInstall.sh $* trap_and_run ./5_BuildCache.sh $* From f4d848dd373b20bb8b533312d9b176538014e81b Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 20 Sep 2024 17:56:07 +0000 Subject: [PATCH 058/134] allow using build cache in scheduled builds --- util/weekly_build/4_SpackInstall.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/4_SpackInstall.sh index 7c98a2c6b..f4d464dd9 100755 --- a/util/weekly_build/4_SpackInstall.sh +++ b/util/weekly_build/4_SpackInstall.sh @@ -10,7 +10,13 @@ set +x . setup.sh set -x -INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS" +if [ "$REUSE_BUILD_CACHE" == YES ]; then + cache_flag="--no-check-signature" +else + cache_flag="--no-cache" +fi + +INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} From e6c6cd093a777d56db8c4ab83cda8faa5928d6b3 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 20 Sep 2024 18:38:21 +0000 Subject: [PATCH 059/134] rename scripts and break out site setups --- ...DirectorySetup.sh => 01_DirectorySetup.sh} | 0 ...2_GetSpackStack.sh => 02_GetSpackStack.sh} | 0 ...tupUnifiedEnv.sh => 03_SetupUnifiedEnv.sh} | 0 .../{4_SpackInstall.sh => 04_SpackInstall.sh} | 0 .../{5_BuildCache.sh => 05_BuildCache.sh} | 0 .../{6_AppTests.sh => 06_AppTests.sh} | 0 .../{7_Cleanup.sh => 07_Cleanup.sh} | 0 util/weekly_build/ShellSetup.sh | 77 +------------------ .../SpackStackBuildCache_AllSteps.sh | 14 ++-- util/weekly_build/sites/acorn.sh | 25 ++++++ util/weekly_build/sites/derecho.sh | 2 + util/weekly_build/sites/discover16.sh | 2 + util/weekly_build/sites/discover17.sh | 2 + util/weekly_build/sites/gaea.sh | 2 + util/weekly_build/sites/hera.sh | 2 + util/weekly_build/sites/hercules.sh | 2 + util/weekly_build/sites/jet.sh | 2 + util/weekly_build/sites/linux.default.sh | 1 + util/weekly_build/sites/narwhal.sh | 2 + util/weekly_build/sites/nautilus.sh | 2 + util/weekly_build/sites/orion.sh | 2 + util/weekly_build/sites/s4.sh | 2 + 22 files changed, 56 insertions(+), 83 deletions(-) rename util/weekly_build/{1_DirectorySetup.sh => 01_DirectorySetup.sh} (100%) rename util/weekly_build/{2_GetSpackStack.sh => 02_GetSpackStack.sh} (100%) rename util/weekly_build/{3_SetupUnifiedEnv.sh => 03_SetupUnifiedEnv.sh} (100%) rename util/weekly_build/{4_SpackInstall.sh => 04_SpackInstall.sh} (100%) rename util/weekly_build/{5_BuildCache.sh => 05_BuildCache.sh} (100%) rename util/weekly_build/{6_AppTests.sh => 06_AppTests.sh} (100%) rename util/weekly_build/{7_Cleanup.sh => 07_Cleanup.sh} (100%) create mode 100644 util/weekly_build/sites/acorn.sh create mode 100644 util/weekly_build/sites/derecho.sh create mode 100644 util/weekly_build/sites/discover16.sh create mode 100644 util/weekly_build/sites/discover17.sh create mode 100644 util/weekly_build/sites/gaea.sh create mode 100644 util/weekly_build/sites/hera.sh create mode 100644 util/weekly_build/sites/hercules.sh create mode 100644 util/weekly_build/sites/jet.sh create mode 100644 util/weekly_build/sites/linux.default.sh create mode 100644 util/weekly_build/sites/narwhal.sh create mode 100644 util/weekly_build/sites/nautilus.sh create mode 100644 util/weekly_build/sites/orion.sh create mode 100644 util/weekly_build/sites/s4.sh diff --git a/util/weekly_build/1_DirectorySetup.sh b/util/weekly_build/01_DirectorySetup.sh similarity index 100% rename from util/weekly_build/1_DirectorySetup.sh rename to util/weekly_build/01_DirectorySetup.sh diff --git a/util/weekly_build/2_GetSpackStack.sh b/util/weekly_build/02_GetSpackStack.sh similarity index 100% rename from util/weekly_build/2_GetSpackStack.sh rename to util/weekly_build/02_GetSpackStack.sh diff --git a/util/weekly_build/3_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupUnifiedEnv.sh similarity index 100% rename from util/weekly_build/3_SetupUnifiedEnv.sh rename to util/weekly_build/03_SetupUnifiedEnv.sh diff --git a/util/weekly_build/4_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh similarity index 100% rename from util/weekly_build/4_SpackInstall.sh rename to util/weekly_build/04_SpackInstall.sh diff --git a/util/weekly_build/5_BuildCache.sh b/util/weekly_build/05_BuildCache.sh similarity index 100% rename from util/weekly_build/5_BuildCache.sh rename to util/weekly_build/05_BuildCache.sh diff --git a/util/weekly_build/6_AppTests.sh b/util/weekly_build/06_AppTests.sh similarity index 100% rename from util/weekly_build/6_AppTests.sh rename to util/weekly_build/06_AppTests.sh diff --git a/util/weekly_build/7_Cleanup.sh b/util/weekly_build/07_Cleanup.sh similarity index 100% rename from util/weekly_build/7_Cleanup.sh rename to util/weekly_build/07_Cleanup.sh diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 2700bdd4e..7f4a73045 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -21,81 +21,6 @@ function spack_install_exe { spack $* | tee -a log.install 2>&1 } -case $PLATFORM in - hercules) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/hercules/build_cache} - ;; - orion) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/orion/build_cache} - ;; - discover16) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu16/build_cache} - ;; - discover17) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu17/build_cache} - ;; - derecho) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} - ;; - acorn) - module load gcc/11.2.0 python/3.11.7 - COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} - function spack_install_exe { -# set +e -# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & -# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & -# wait -# rc=$? -# set -e -# cat spack-build-cache-${RUNID}* -# return $rc -## cp ${SPACK_STACK_DIR:?}/util/acorn/{build.pbs,spackinstall.sh} ${SPACK_ENV}/. -## /opt/pbs/bin/qsub -Wblock=true ${SPACK_ENV}/build.pbs -## spack $* | tee -a log.install 2>&1 - shift 1 - ${SPACK_STACK_DIR}/util/parallel_install.sh 3 4 $* - } - PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" - PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" - function alert_cmd { - module purge # annoying libstdc++ issue - mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") - } - TEST_UFSWM=ON - ;; - gaea) - COMPILERS=${COMPILERS:-"intel"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/ncrc/proj/epic/spack-stack/build_cache} - ;; - hera) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/scratch1/NCEPDEV/nems/role.epic/spack-stack/build_cache} - ;; - jet) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/build_cache} - ;; - narwhal) - COMPILERS=${COMPILERS:-"intel gcc"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} - ;; - nautilus) - COMPILERS=${COMPILERS:-"intel"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} - ;; - s4) - COMPILERS=${COMPILERS:-"intel"} - BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/data/prod/jedi/spack-stack/build_cache} - ;; - linux.default) - COMPILERS=${COMPILERS:-"gcc"} - -esac +. $(dirname $0)/sites/${PLATFORM}.sh echo "Build cache target directory: ${BUILD_CACHE_DIR?'BUILD_CACHE_DIR must be set!'}" diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index df04a5e58..4d26df07d 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -16,10 +16,10 @@ function trap_and_run { $* } -trap_and_run ./1_DirectorySetup.sh $* -trap_and_run ./2_GetSpackStack.sh $* -trap_and_run ./3_SetupUnifiedEnv.sh $* -trap_and_run ./4_SpackInstall.sh $* -trap_and_run ./5_BuildCache.sh $* -trap_and_run ./6_AppTests.sh $* -trap_and_run ./7_Cleanup.sh $* +trap_and_run ./01_DirectorySetup.sh $* +trap_and_run ./02_GetSpackStack.sh $* +trap_and_run ./03_SetupUnifiedEnv.sh $* +trap_and_run ./04_SpackInstall.sh $* +trap_and_run ./05_BuildCache.sh $* +trap_and_run ./06_AppTests.sh $* +trap_and_run ./07_Cleanup.sh $* diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh new file mode 100644 index 000000000..3050d0564 --- /dev/null +++ b/util/weekly_build/sites/acorn.sh @@ -0,0 +1,25 @@ +module load gcc/11.2.0 python/3.11.7 +COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} +function spack_install_exe { +# set +e +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & +# wait +# rc=$? +# set -e +# cat spack-build-cache-${RUNID}* +# return $rc +## cp ${SPACK_STACK_DIR:?}/util/acorn/{build.pbs,spackinstall.sh} ${SPACK_ENV}/. +## /opt/pbs/bin/qsub -Wblock=true ${SPACK_ENV}/build.pbs +## spack $* | tee -a log.install 2>&1 + shift 1 + ${SPACK_STACK_DIR}/util/parallel_install.sh 3 4 $* +} +PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" +PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" +function alert_cmd { + module purge # annoying libstdc++ issue + mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") +} +TEST_UFSWM=ON diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh new file mode 100644 index 000000000..bde573785 --- /dev/null +++ b/util/weekly_build/sites/derecho.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh new file mode 100644 index 000000000..2d3f0fa98 --- /dev/null +++ b/util/weekly_build/sites/discover16.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu16/build_cache} diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh new file mode 100644 index 000000000..b8a2a57ac --- /dev/null +++ b/util/weekly_build/sites/discover17.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu17/build_cache} diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh new file mode 100644 index 000000000..d9d692e8b --- /dev/null +++ b/util/weekly_build/sites/gaea.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/ncrc/proj/epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh new file mode 100644 index 000000000..513421227 --- /dev/null +++ b/util/weekly_build/sites/hera.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/scratch1/NCEPDEV/nems/role.epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh new file mode 100644 index 000000000..7c1d29cfe --- /dev/null +++ b/util/weekly_build/sites/hercules.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/hercules/build_cache} diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh new file mode 100644 index 000000000..cf991daf3 --- /dev/null +++ b/util/weekly_build/sites/jet.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/linux.default.sh b/util/weekly_build/sites/linux.default.sh new file mode 100644 index 000000000..c044afced --- /dev/null +++ b/util/weekly_build/sites/linux.default.sh @@ -0,0 +1 @@ +COMPILERS=${COMPILERS:-"gcc"} diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh new file mode 100644 index 000000000..2c5705d21 --- /dev/null +++ b/util/weekly_build/sites/narwhal.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh new file mode 100644 index 000000000..7bcf4c9cd --- /dev/null +++ b/util/weekly_build/sites/nautilus.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh new file mode 100644 index 000000000..4e9a6a86f --- /dev/null +++ b/util/weekly_build/sites/orion.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel gcc"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/orion/build_cache} diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh new file mode 100644 index 000000000..0854328b4 --- /dev/null +++ b/util/weekly_build/sites/s4.sh @@ -0,0 +1,2 @@ +COMPILERS=${COMPILERS:-"intel"} +BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/data/prod/jedi/spack-stack/build_cache} From 5202e44a7862fb554e3d83a57dca6a283a8a143f Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Fri, 20 Sep 2024 21:00:37 +0000 Subject: [PATCH 060/134] use one cache dir for each site and rebuild cache index --- configs/sites/tier1/acorn/mirrors.yaml | 6 +++--- configs/sites/tier1/aws-pcluster/mirrors.yaml | 6 +++--- configs/sites/tier1/casper/mirrors.yaml | 6 +++--- configs/sites/tier1/derecho/mirrors.yaml | 6 +++--- configs/sites/tier1/discover-scu16/mirrors.yaml | 6 +++--- configs/sites/tier1/discover-scu17/mirrors.yaml | 6 +++--- configs/sites/tier1/gaea-c5/mirrors.yaml | 6 +++--- configs/sites/tier1/gaea-c6/mirrors.yaml | 6 +++--- configs/sites/tier1/hera/mirrors.yaml | 6 +++--- configs/sites/tier1/hercules/mirrors.yaml | 6 +++--- configs/sites/tier1/jet/mirrors.yaml | 6 +++--- configs/sites/tier1/narwhal/mirrors.yaml | 6 +++--- configs/sites/tier1/nautilus/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-aws/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-azure/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-gcloud/mirrors.yaml | 6 +++--- configs/sites/tier1/orion/mirrors.yaml | 6 +++--- configs/sites/tier1/s4/mirrors.yaml | 6 +++--- configs/sites/tier1/sandy/mirrors.yaml | 6 +++--- util/weekly_build/05_BuildCache.sh | 3 ++- util/weekly_build/ShellSetup.sh | 2 +- util/weekly_build/sites/acorn.sh | 1 - util/weekly_build/sites/derecho.sh | 1 - util/weekly_build/sites/discover16.sh | 1 - util/weekly_build/sites/discover17.sh | 1 - util/weekly_build/sites/gaea.sh | 1 - util/weekly_build/sites/hera.sh | 1 - util/weekly_build/sites/hercules.sh | 1 - util/weekly_build/sites/jet.sh | 1 - util/weekly_build/sites/narwhal.sh | 1 - util/weekly_build/sites/nautilus.sh | 1 - util/weekly_build/sites/orion.sh | 1 - util/weekly_build/sites/s4.sh | 1 - 33 files changed, 60 insertions(+), 71 deletions(-) diff --git a/configs/sites/tier1/acorn/mirrors.yaml b/configs/sites/tier1/acorn/mirrors.yaml index 82239d643..bfdc2842c 100644 --- a/configs/sites/tier1/acorn/mirrors.yaml +++ b/configs/sites/tier1/acorn/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/source-cache + url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/source-cache + url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/aws-pcluster/mirrors.yaml b/configs/sites/tier1/aws-pcluster/mirrors.yaml index 52d6f0e4d..5bfb083b9 100644 --- a/configs/sites/tier1/aws-pcluster/mirrors.yaml +++ b/configs/sites/tier1/aws-pcluster/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source-efs: + local-cache-efs: fetch: - url: file:///mnt/experiments-efs/spack-stack/source-cache + url: file:///mnt/experiments-efs/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///mnt/experiments-efs/spack-stack/source-cache + url: file:///mnt/experiments-efs/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/casper/mirrors.yaml b/configs/sites/tier1/casper/mirrors.yaml index b42871aa0..abe30ea6d 100644 --- a/configs/sites/tier1/casper/mirrors.yaml +++ b/configs/sites/tier1/casper/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/derecho/mirrors.yaml b/configs/sites/tier1/derecho/mirrors.yaml index b42871aa0..abe30ea6d 100644 --- a/configs/sites/tier1/derecho/mirrors.yaml +++ b/configs/sites/tier1/derecho/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/discover-scu16/mirrors.yaml b/configs/sites/tier1/discover-scu16/mirrors.yaml index 596a615bf..4fc4ae78e 100644 --- a/configs/sites/tier1/discover-scu16/mirrors.yaml +++ b/configs/sites/tier1/discover-scu16/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///discover/swdev/jcsda/spack-stack/source-cache + url: file:///discover/swdev/jcsda/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///discover/swdev/jcsda/spack-stack/source-cache + url: file:///discover/swdev/jcsda/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/discover-scu17/mirrors.yaml b/configs/sites/tier1/discover-scu17/mirrors.yaml index 596a615bf..4fc4ae78e 100644 --- a/configs/sites/tier1/discover-scu17/mirrors.yaml +++ b/configs/sites/tier1/discover-scu17/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///discover/swdev/jcsda/spack-stack/source-cache + url: file:///discover/swdev/jcsda/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///discover/swdev/jcsda/spack-stack/source-cache + url: file:///discover/swdev/jcsda/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/gaea-c5/mirrors.yaml b/configs/sites/tier1/gaea-c5/mirrors.yaml index bb9e96768..ca54a6946 100644 --- a/configs/sites/tier1/gaea-c5/mirrors.yaml +++ b/configs/sites/tier1/gaea-c5/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/gaea-c6/mirrors.yaml b/configs/sites/tier1/gaea-c6/mirrors.yaml index bb9e96768..ca54a6946 100644 --- a/configs/sites/tier1/gaea-c6/mirrors.yaml +++ b/configs/sites/tier1/gaea-c6/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/hera/mirrors.yaml b/configs/sites/tier1/hera/mirrors.yaml index f6c5a8d1e..e883650ba 100644 --- a/configs/sites/tier1/hera/mirrors.yaml +++ b/configs/sites/tier1/hera/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/hercules/mirrors.yaml b/configs/sites/tier1/hercules/mirrors.yaml index 97f382e8e..c49485871 100644 --- a/configs/sites/tier1/hercules/mirrors.yaml +++ b/configs/sites/tier1/hercules/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///work/noaa/epic/role-epic/spack-stack/source-cache + url: file:///work/noaa/epic/role-epic/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///work/noaa/epic/role-epic/spack-stack/source-cache + url: file:///work/noaa/epic/role-epic/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/jet/mirrors.yaml b/configs/sites/tier1/jet/mirrors.yaml index 6198725df..6c4aef52e 100644 --- a/configs/sites/tier1/jet/mirrors.yaml +++ b/configs/sites/tier1/jet/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/source-cache + url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/source-cache + url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 709f9096c..bb6ee10bd 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 709f9096c..bb6ee10bd 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-aws/mirrors.yaml b/configs/sites/tier1/noaa-aws/mirrors.yaml index 3d247981f..7393a2b1a 100644 --- a/configs/sites/tier1/noaa-aws/mirrors.yaml +++ b/configs/sites/tier1/noaa-aws/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-azure/mirrors.yaml b/configs/sites/tier1/noaa-azure/mirrors.yaml index 3d247981f..7393a2b1a 100644 --- a/configs/sites/tier1/noaa-azure/mirrors.yaml +++ b/configs/sites/tier1/noaa-azure/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-gcloud/mirrors.yaml b/configs/sites/tier1/noaa-gcloud/mirrors.yaml index 3d247981f..7393a2b1a 100644 --- a/configs/sites/tier1/noaa-gcloud/mirrors.yaml +++ b/configs/sites/tier1/noaa-gcloud/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/source-cache + url: file:///contrib/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/orion/mirrors.yaml b/configs/sites/tier1/orion/mirrors.yaml index 97f382e8e..c49485871 100644 --- a/configs/sites/tier1/orion/mirrors.yaml +++ b/configs/sites/tier1/orion/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///work/noaa/epic/role-epic/spack-stack/source-cache + url: file:///work/noaa/epic/role-epic/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///work/noaa/epic/role-epic/spack-stack/source-cache + url: file:///work/noaa/epic/role-epic/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/s4/mirrors.yaml b/configs/sites/tier1/s4/mirrors.yaml index 152750f81..2a7037264 100644 --- a/configs/sites/tier1/s4/mirrors.yaml +++ b/configs/sites/tier1/s4/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///data/prod/jedi/spack-stack/source-cache + url: file:///data/prod/jedi/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///data/prod/jedi/spack-stack/source-cache + url: file:///data/prod/jedi/spack-stack/cache access_pair: - null - null diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml index c48f33bf1..b63a47e30 100644 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ b/configs/sites/tier1/sandy/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-source: + local-cache: fetch: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///gpfs/fs1/neptune/spack-stack/cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///gpfs/fs1/neptune/spack-stack/cache access_pair: - null - null diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh index 45982ff01..5210bb514 100755 --- a/util/weekly_build/05_BuildCache.sh +++ b/util/weekly_build/05_BuildCache.sh @@ -13,5 +13,6 @@ set -x for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack buildcache push --unsigned --force ${BUILD_CACHE_DIR?"BUILD_CACHE_DIR must be set!"} $PACKAGES_TO_INSTALL + spack buildcache push --unsigned --force ${BUILD_CACHE:-local-binary} $PACKAGES_TO_INSTALL + spack buildcache rebuild-index ${BUILD_CACHE:-local-binary} done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 7f4a73045..ff9ba802e 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -23,4 +23,4 @@ function spack_install_exe { . $(dirname $0)/sites/${PLATFORM}.sh -echo "Build cache target directory: ${BUILD_CACHE_DIR?'BUILD_CACHE_DIR must be set!'}" +echo "Build cache target name/directory: ${BUILD_CACHE?'BUILD_CACHE must be set (name of configured mirror or directory path!'}" diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index 3050d0564..8fb170d17 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -1,6 +1,5 @@ module load gcc/11.2.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/lfs/h1/emc/nceplibs/noscrub/spack-stack/build_cache} function spack_install_exe { # set +e # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh index bde573785..1e7dc8450 100644 --- a/util/weekly_build/sites/derecho.sh +++ b/util/weekly_build/sites/derecho.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/glade/work/epicufsrt/contrib/spack-stack/derecho/build_cache} diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh index 2d3f0fa98..1e7dc8450 100644 --- a/util/weekly_build/sites/discover16.sh +++ b/util/weekly_build/sites/discover16.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu16/build_cache} diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh index b8a2a57ac..1e7dc8450 100644 --- a/util/weekly_build/sites/discover17.sh +++ b/util/weekly_build/sites/discover17.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/gpfsm/dswdev/jcsda/spack-stack/scu17/build_cache} diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh index d9d692e8b..fe7e69664 100644 --- a/util/weekly_build/sites/gaea.sh +++ b/util/weekly_build/sites/gaea.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/ncrc/proj/epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh index 513421227..1e7dc8450 100644 --- a/util/weekly_build/sites/hera.sh +++ b/util/weekly_build/sites/hera.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/scratch1/NCEPDEV/nems/role.epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh index 7c1d29cfe..1e7dc8450 100644 --- a/util/weekly_build/sites/hercules.sh +++ b/util/weekly_build/sites/hercules.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/hercules/build_cache} diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh index cf991daf3..1e7dc8450 100644 --- a/util/weekly_build/sites/jet.sh +++ b/util/weekly_build/sites/jet.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/build_cache} diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh index 2c5705d21..1e7dc8450 100644 --- a/util/weekly_build/sites/narwhal.sh +++ b/util/weekly_build/sites/narwhal.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh index 7bcf4c9cd..fe7e69664 100644 --- a/util/weekly_build/sites/nautilus.sh +++ b/util/weekly_build/sites/nautilus.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build_cache} diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh index 4e9a6a86f..1e7dc8450 100644 --- a/util/weekly_build/sites/orion.sh +++ b/util/weekly_build/sites/orion.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel gcc"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/work/noaa/epic/role-epic/spack-stack/orion/build_cache} diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh index 0854328b4..fe7e69664 100644 --- a/util/weekly_build/sites/s4.sh +++ b/util/weekly_build/sites/s4.sh @@ -1,2 +1 @@ COMPILERS=${COMPILERS:-"intel"} -BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/data/prod/jedi/spack-stack/build_cache} From 6c61957ea37475ac8d75e6d18e6ed28893e977aa Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 24 Sep 2024 07:45:02 -0600 Subject: [PATCH 061/134] Increase timeout of .github/workflows/nautilus-nightly-oneapi.yaml to 720 minutes --- .github/workflows/nautilus-nightly-oneapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index 5c8b33f9f..a7062f6f1 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -25,7 +25,7 @@ jobs: nightly-build-oneapi: runs-on: ['self-hosted','nautilus-heinzell'] # Set timeout to 12hrs - #timeout-minutes: 720 + timeout-minutes: 720 steps: - uses: actions/checkout@v4 with: From 71e5a2c65c12c47a65c0e3123e6cfe28ad23a32d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 24 Sep 2024 09:05:10 -0600 Subject: [PATCH 062/134] Add .github/workflows/nautilus-nightly-intel.yaml and build only base-env in unified-dev for testing --- .github/workflows/nautilus-nightly-intel.yaml | 41 +++++++++++++ .../workflows/nautilus-nightly-oneapi.yaml | 8 +-- configs/templates/unified-dev/spack.yaml | 57 ++++++++++--------- util/weekly_build/03_SetupEnv.sh | 2 + 4 files changed, 76 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/nautilus-nightly-intel.yaml diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml new file mode 100644 index 000000000..e9a55b191 --- /dev/null +++ b/.github/workflows/nautilus-nightly-intel.yaml @@ -0,0 +1,41 @@ +name: nautilus unified-env intel + +on: + workflow_dispatch: + pull_request: + branches: + - develop + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + BASEDIR: /p/work2/heinzell/spack-stack-nightly + PLATFORM: nautilus + COMPILERS: intel + +jobs: + + nightly-build-intel: + runs-on: ['self-hosted','nautilus-heinzell'] + # Set timeout to 12hrs + timeout-minutes: 720 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + clean: true + - name: 01_DirectorySetup + uses: ./.github/actions/Nightly_01_DirectorySetup + - name: 02_GetSpackStackDevelop + uses: ./.github/actions/Nightly_02_GetSpackStackDevelop + - name: 03_SetupUnifiedEnv + uses: ./.github/actions/Nightly_03_SetupUnifiedEnv + #- name: 04_SpackInstall + # uses: ./.github/actions/Nightly_04_SpackInstall diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index a7062f6f1..671c7555d 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -1,4 +1,4 @@ -name: nautilus heinzell auto ci +name: nautilus unified-env oneapi on: workflow_dispatch: @@ -18,7 +18,7 @@ defaults: env: BASEDIR: /p/work2/heinzell/spack-stack-nightly PLATFORM: nautilus - COMPILER: oneapi + COMPILERS: oneapi jobs: @@ -37,5 +37,5 @@ jobs: uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - name: 03_SetupUnifiedEnv uses: ./.github/actions/Nightly_03_SetupUnifiedEnv - - name: 04_SpackInstall - uses: ./.github/actions/Nightly_04_SpackInstall + #- name: 04_SpackInstall + # uses: ./.github/actions/Nightly_04_SpackInstall diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index a05ed4a75..6420943f4 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -8,34 +8,35 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - ewok-env +ecflow ~cylc - - ai-env - - geos-gcm-env ^esmf@=8.6.1 - - global-workflow-env ^esmf@=8.6.1 - - gmao-swell-env - - gsi-env ^esmf@=8.6.1 - - jedi-fv3-env - - jedi-geos-env ^esmf@=8.6.1 - - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - jedi-tools-env - - jedi-ufs-env ^esmf@=8.6.1 - - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 - - soca-env - - ufs-srw-app-env ^esmf@=8.6.1 - - ufs-weather-model-env ^esmf@=8.6.1 - - # Various crtm tags (list all to avoid duplicate packages) - - crtm@2.4.0.1 - - crtm@v2.4.1-jedi - - # Various esmf tags (list all to avoid duplicate packages) - - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 - - # MADIS for WCOSS2 decoders. - - madis@4.5 + - base-env + #- ewok-env +ecflow ~cylc + #- ai-env + #- geos-gcm-env ^esmf@=8.6.1 + #- global-workflow-env ^esmf@=8.6.1 + #- gmao-swell-env + #- gsi-env ^esmf@=8.6.1 + #- jedi-fv3-env + #- jedi-geos-env ^esmf@=8.6.1 + #- jedi-mpas-env + #- jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + #- jedi-tools-env + #- jedi-ufs-env ^esmf@=8.6.1 + #- jedi-um-env + #- neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + #- soca-env + #- ufs-srw-app-env ^esmf@=8.6.1 + #- ufs-weather-model-env ^esmf@=8.6.1 + # + ## Various crtm tags (list all to avoid duplicate packages) + #- crtm@2.4.0.1 + #- crtm@v2.4.1-jedi + # + ## Various esmf tags (list all to avoid duplicate packages) + #- esmf@=8.6.1 snapshot=none + #- esmf@=8.7.0b11 snapshot=b11 + # + ## MADIS for WCOSS2 decoders. + #- madis@4.5 specs: - matrix: diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 3479ed7ec..303240dc4 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -11,6 +11,8 @@ set +x set -x for compiler in $COMPILERS; do + echo "Setting up environment build-${compiler/@/-} using template unified-dev" + #echo "Setting up environment build-${compiler/@/-} using template unified-dev" rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} From 18c71335ceb135f9bf155587134ac746e440ed25 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 24 Sep 2024 11:20:14 -0600 Subject: [PATCH 063/134] Enable step 4 of nightly tests on Nautilus with Intel and oneAPI --- .github/workflows/nautilus-nightly-intel.yaml | 4 ++-- .github/workflows/nautilus-nightly-oneapi.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml index e9a55b191..7dbf11bec 100644 --- a/.github/workflows/nautilus-nightly-intel.yaml +++ b/.github/workflows/nautilus-nightly-intel.yaml @@ -37,5 +37,5 @@ jobs: uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - name: 03_SetupUnifiedEnv uses: ./.github/actions/Nightly_03_SetupUnifiedEnv - #- name: 04_SpackInstall - # uses: ./.github/actions/Nightly_04_SpackInstall + - name: 04_SpackInstall + uses: ./.github/actions/Nightly_04_SpackInstall diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index 671c7555d..df7b708a5 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -37,5 +37,5 @@ jobs: uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - name: 03_SetupUnifiedEnv uses: ./.github/actions/Nightly_03_SetupUnifiedEnv - #- name: 04_SpackInstall - # uses: ./.github/actions/Nightly_04_SpackInstall + - name: 04_SpackInstall + uses: ./.github/actions/Nightly_04_SpackInstall From f5a575b7d61eb1485b6c9798edbdc3f21e1abfb2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 24 Sep 2024 16:06:37 -0600 Subject: [PATCH 064/134] Remove piping to log files in util/weekly_build/03_SetupEnv.sh and util/weekly_build/04_SpackInstall.sh --- util/weekly_build/03_SetupEnv.sh | 3 ++- util/weekly_build/04_SpackInstall.sh | 8 ++++---- util/weekly_build/ShellSetup.sh | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 303240dc4..ccaf562fe 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -18,9 +18,10 @@ for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" - # Check for duplicates and fail before doing the "real" concretization with test deps: + # Check for duplicates and fail before doing the "real" concretization: spack concretize --fresh 2>&1 | tee log.concretize ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf + spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index 7c98a2c6b..34711fcc8 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -16,12 +16,12 @@ for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . if [ -z "$PACKAGES_TO_INSTALL" ]; then - spack fetch --missing 2>&1 | tee log.fetch + spack fetch --missing # 2>&1 | tee log.fetch else - spack fetch --missing --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch + spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch fi # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.test.install + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL # 2>&1 | tee log.install done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 24767ea78..61243d933 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -18,7 +18,8 @@ function alert_cmd { } function spack_install_exe { - spack $* | tee -a log.install 2>&1 + #spack $* | tee -a log.install 2>&1 + spack $* } case $PLATFORM in From 720e0ec496c8486c7b34909d285fe029653e746a Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 24 Sep 2024 23:16:48 +0000 Subject: [PATCH 065/134] clone faster --- util/weekly_build/02_GetSpackStack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/02_GetSpackStack.sh b/util/weekly_build/02_GetSpackStack.sh index 368d13522..5fe77f7cc 100755 --- a/util/weekly_build/02_GetSpackStack.sh +++ b/util/weekly_build/02_GetSpackStack.sh @@ -9,5 +9,5 @@ if [ -d $RUNID ]; then cd $RUNID git pull else - git clone --recurse-submodules ${SPACK_STACK_URL:-https://github.com/AlexanderRichert-NOAA/spack-stack} -b ${SPACK_STACK_BRANCH:-weekly_build} $RUNID + git clone --depth 1 --recurse-submodules --shallow-submodules ${SPACK_STACK_URL:-https://github.com/AlexanderRichert-NOAA/spack-stack} -b ${SPACK_STACK_BRANCH:-weekly_build} $RUNID fi From d4597d74da8fb6471da36f315a0a287e6fdaf115 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 24 Sep 2024 23:18:57 +0000 Subject: [PATCH 066/134] remove redundant redirect for install steps --- util/weekly_build/04_SpackInstall.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index f4d464dd9..c0928ac84 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -27,7 +27,7 @@ for compiler in $COMPILERS; do spack fetch --missing --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch fi # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST 2>&1 | tee log.test.install + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL 2>&1 | tee log.install + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL done From 979607d0cdd6f7edb2fa113d090cb03280f23070 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 24 Sep 2024 21:07:29 -0600 Subject: [PATCH 067/134] Comment out default definition of PACKAGES_TO_TEST --- util/weekly_build/ShellSetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 61243d933..c12f4cee7 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -11,7 +11,7 @@ if [ ${RUNDIR::1} != "/" ]; then exit 1 fi -PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} +#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} function alert_cmd { echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." From 4709452ccc91ad43d29ab6bb8da91b5f6cf9bde7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 25 Sep 2024 15:36:41 -0600 Subject: [PATCH 068/134] Try spack fetch without --missing --- util/weekly_build/04_SpackInstall.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index 34711fcc8..ac396de1d 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -15,11 +15,12 @@ INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - if [ -z "$PACKAGES_TO_INSTALL" ]; then - spack fetch --missing # 2>&1 | tee log.fetch - else - spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch - fi + spack fetch # 2>&1 | tee log.fetch + #if [ -z "$PACKAGES_TO_INSTALL" ]; then + # spack fetch --missing # 2>&1 | tee log.fetch + #else + # spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch + #fi # Just install the packages we're testing (+dependencies): spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install # Install the rest of the stack as usual: From b4bcf8fa70f4e7a750481b003f248751a9bb6a08 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 26 Sep 2024 16:00:20 -0600 Subject: [PATCH 069/134] Add logic to extract path to local source mirror and comment out 'spack fetch' --- util/weekly_build/03_SetupEnv.sh | 5 +++++ util/weekly_build/04_SpackInstall.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index ccaf562fe..a6d360ed5 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -24,4 +24,9 @@ for compiler in $COMPILERS; do spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test + # Get path to local-source mirror from spack site config + SOURCE_MIRROR=`spack mirror list | grep local-source` + SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} + # Download all source codes needed to build the environment + spack mirror create -a -d ${SOURCE_MIRROR} done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index ac396de1d..f6edac28e 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -15,7 +15,7 @@ INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack fetch # 2>&1 | tee log.fetch + #spack fetch # 2>&1 | tee log.fetch #if [ -z "$PACKAGES_TO_INSTALL" ]; then # spack fetch --missing # 2>&1 | tee log.fetch #else From ff17223cee098ff9c87d09ca2a6321b46af03e7f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 27 Sep 2024 10:27:56 -0600 Subject: [PATCH 070/134] Configure oneapi@2024.2.1 for Narwhal --- configs/sites/tier1/narwhal/compilers.yaml | 32 +++++++++++++++++++ .../sites/tier1/narwhal/packages_oneapi.yaml | 32 +++++++++++++++++++ doc/source/PreConfiguredSites.rst | 30 ++++++++++++++++- 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 configs/sites/tier1/narwhal/packages_oneapi.yaml diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 1b7c23dcf..5e2e17c11 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -21,6 +21,38 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: + spec: oneapi@2024.2.1 + paths: + #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx + #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx + #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.3.3 + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + - ifort/2024.2.1 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' + append_path: + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml new file mode 100644 index 000000000..9b032dcf1 --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -0,0 +1,32 @@ +packages: + all: + compiler:: [oneapi@2024.2.1,gcc@10.3.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + externals: + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 97b532daa..0ea8e3a7b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -63,7 +63,7 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | | U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -169,6 +169,34 @@ The following is required for building new spack environments with Intel on this module unload cray-libsci module load cray-libsci/23.05.1.4 +The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! + +.. code-block:: console + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-intel/8.3.3 + module unload intel + module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles + module load tbb/2021.13 + module load compiler-rt/2024.2.1 + module load compiler/2024.2.1 + module load ifort/2024.2.1 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console From d4d17672625c7fb9cfe1755f68358106c6801952 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 30 Sep 2024 20:21:14 +0000 Subject: [PATCH 071/134] testing shortened mirrors.yaml on acorn --- configs/sites/tier1/acorn/mirrors.yaml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/configs/sites/tier1/acorn/mirrors.yaml b/configs/sites/tier1/acorn/mirrors.yaml index bfdc2842c..366f74171 100644 --- a/configs/sites/tier1/acorn/mirrors.yaml +++ b/configs/sites/tier1/acorn/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-cache: - fetch: - url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache + local-binary: file:///lfs/h1/emc/nceplibs/noscrub/spack-stack/cache From 4107978ce628436d548312baaa6baa6982d7d36c Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 30 Sep 2024 20:22:25 +0000 Subject: [PATCH 072/134] remove source_cache from acorn/config.yaml --- configs/sites/tier1/acorn/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/sites/tier1/acorn/config.yaml b/configs/sites/tier1/acorn/config.yaml index 9f64f0688..56e793047 100644 --- a/configs/sites/tier1/acorn/config.yaml +++ b/configs/sites/tier1/acorn/config.yaml @@ -1,4 +1,3 @@ config: build_jobs: 6 build_stage: $tempdir/$user/spack-stage - source_cache: /lfs/h1/emc/nceplibs/noscrub/spack-stack/source_cache From 2092fc4c6e2d830971296f36284863bb9755af62 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 30 Sep 2024 20:23:04 +0000 Subject: [PATCH 073/134] revert mirrors.yaml changes for all sites but acorn --- configs/sites/tier1/aws-pcluster/mirrors.yaml | 6 +++--- configs/sites/tier1/casper/mirrors.yaml | 6 +++--- configs/sites/tier1/derecho/mirrors.yaml | 6 +++--- configs/sites/tier1/discover-scu16/mirrors.yaml | 6 +++--- configs/sites/tier1/discover-scu17/mirrors.yaml | 6 +++--- configs/sites/tier1/gaea-c5/mirrors.yaml | 6 +++--- configs/sites/tier1/gaea-c6/mirrors.yaml | 6 +++--- configs/sites/tier1/hera/mirrors.yaml | 6 +++--- configs/sites/tier1/hercules/mirrors.yaml | 6 +++--- configs/sites/tier1/jet/mirrors.yaml | 6 +++--- configs/sites/tier1/narwhal/mirrors.yaml | 6 +++--- configs/sites/tier1/nautilus/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-aws/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-azure/mirrors.yaml | 6 +++--- configs/sites/tier1/noaa-gcloud/mirrors.yaml | 6 +++--- configs/sites/tier1/orion/mirrors.yaml | 6 +++--- configs/sites/tier1/s4/mirrors.yaml | 6 +++--- configs/sites/tier1/sandy/mirrors.yaml | 6 +++--- 18 files changed, 54 insertions(+), 54 deletions(-) diff --git a/configs/sites/tier1/aws-pcluster/mirrors.yaml b/configs/sites/tier1/aws-pcluster/mirrors.yaml index 5bfb083b9..52d6f0e4d 100644 --- a/configs/sites/tier1/aws-pcluster/mirrors.yaml +++ b/configs/sites/tier1/aws-pcluster/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache-efs: + local-source-efs: fetch: - url: file:///mnt/experiments-efs/spack-stack/cache + url: file:///mnt/experiments-efs/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///mnt/experiments-efs/spack-stack/cache + url: file:///mnt/experiments-efs/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/casper/mirrors.yaml b/configs/sites/tier1/casper/mirrors.yaml index abe30ea6d..b42871aa0 100644 --- a/configs/sites/tier1/casper/mirrors.yaml +++ b/configs/sites/tier1/casper/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///glade/work/epicufsrt/contrib/spack-stack/cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///glade/work/epicufsrt/contrib/spack-stack/cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/derecho/mirrors.yaml b/configs/sites/tier1/derecho/mirrors.yaml index abe30ea6d..b42871aa0 100644 --- a/configs/sites/tier1/derecho/mirrors.yaml +++ b/configs/sites/tier1/derecho/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///glade/work/epicufsrt/contrib/spack-stack/cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///glade/work/epicufsrt/contrib/spack-stack/cache + url: file:///glade/work/epicufsrt/contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/discover-scu16/mirrors.yaml b/configs/sites/tier1/discover-scu16/mirrors.yaml index 4fc4ae78e..596a615bf 100644 --- a/configs/sites/tier1/discover-scu16/mirrors.yaml +++ b/configs/sites/tier1/discover-scu16/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///discover/swdev/jcsda/spack-stack/cache + url: file:///discover/swdev/jcsda/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///discover/swdev/jcsda/spack-stack/cache + url: file:///discover/swdev/jcsda/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/discover-scu17/mirrors.yaml b/configs/sites/tier1/discover-scu17/mirrors.yaml index 4fc4ae78e..596a615bf 100644 --- a/configs/sites/tier1/discover-scu17/mirrors.yaml +++ b/configs/sites/tier1/discover-scu17/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///discover/swdev/jcsda/spack-stack/cache + url: file:///discover/swdev/jcsda/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///discover/swdev/jcsda/spack-stack/cache + url: file:///discover/swdev/jcsda/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/gaea-c5/mirrors.yaml b/configs/sites/tier1/gaea-c5/mirrors.yaml index ca54a6946..bb9e96768 100644 --- a/configs/sites/tier1/gaea-c5/mirrors.yaml +++ b/configs/sites/tier1/gaea-c5/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/gaea-c6/mirrors.yaml b/configs/sites/tier1/gaea-c6/mirrors.yaml index ca54a6946..bb9e96768 100644 --- a/configs/sites/tier1/gaea-c6/mirrors.yaml +++ b/configs/sites/tier1/gaea-c6/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/cache + url: file:///lustre/f2/dev/wpo/role.epic/contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/hera/mirrors.yaml b/configs/sites/tier1/hera/mirrors.yaml index e883650ba..f6c5a8d1e 100644 --- a/configs/sites/tier1/hera/mirrors.yaml +++ b/configs/sites/tier1/hera/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/hercules/mirrors.yaml b/configs/sites/tier1/hercules/mirrors.yaml index c49485871..97f382e8e 100644 --- a/configs/sites/tier1/hercules/mirrors.yaml +++ b/configs/sites/tier1/hercules/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///work/noaa/epic/role-epic/spack-stack/cache + url: file:///work/noaa/epic/role-epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///work/noaa/epic/role-epic/spack-stack/cache + url: file:///work/noaa/epic/role-epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/jet/mirrors.yaml b/configs/sites/tier1/jet/mirrors.yaml index 6c4aef52e..6198725df 100644 --- a/configs/sites/tier1/jet/mirrors.yaml +++ b/configs/sites/tier1/jet/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/cache + url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/cache + url: file:///mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index bb6ee10bd..709f9096c 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index bb6ee10bd..709f9096c 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-aws/mirrors.yaml b/configs/sites/tier1/noaa-aws/mirrors.yaml index 7393a2b1a..3d247981f 100644 --- a/configs/sites/tier1/noaa-aws/mirrors.yaml +++ b/configs/sites/tier1/noaa-aws/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-azure/mirrors.yaml b/configs/sites/tier1/noaa-azure/mirrors.yaml index 7393a2b1a..3d247981f 100644 --- a/configs/sites/tier1/noaa-azure/mirrors.yaml +++ b/configs/sites/tier1/noaa-azure/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/noaa-gcloud/mirrors.yaml b/configs/sites/tier1/noaa-gcloud/mirrors.yaml index 7393a2b1a..3d247981f 100644 --- a/configs/sites/tier1/noaa-gcloud/mirrors.yaml +++ b/configs/sites/tier1/noaa-gcloud/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///contrib/spack-stack/cache + url: file:///contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/orion/mirrors.yaml b/configs/sites/tier1/orion/mirrors.yaml index c49485871..97f382e8e 100644 --- a/configs/sites/tier1/orion/mirrors.yaml +++ b/configs/sites/tier1/orion/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///work/noaa/epic/role-epic/spack-stack/cache + url: file:///work/noaa/epic/role-epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///work/noaa/epic/role-epic/spack-stack/cache + url: file:///work/noaa/epic/role-epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/s4/mirrors.yaml b/configs/sites/tier1/s4/mirrors.yaml index 2a7037264..152750f81 100644 --- a/configs/sites/tier1/s4/mirrors.yaml +++ b/configs/sites/tier1/s4/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///data/prod/jedi/spack-stack/cache + url: file:///data/prod/jedi/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///data/prod/jedi/spack-stack/cache + url: file:///data/prod/jedi/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml index b63a47e30..c48f33bf1 100644 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ b/configs/sites/tier1/sandy/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: - local-cache: + local-source: fetch: - url: file:///gpfs/fs1/neptune/spack-stack/cache + url: file:///gpfs/fs1/neptune/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///gpfs/fs1/neptune/spack-stack/cache + url: file:///gpfs/fs1/neptune/spack-stack/source-cache access_pair: - null - null From 8eeafda436e90b8f719db7a9864ea33c0afefc0a Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Mon, 30 Sep 2024 20:57:33 +0000 Subject: [PATCH 074/134] update source & build cache logic --- util/weekly_build/04_SpackInstall.sh | 7 ++++--- util/weekly_build/05_BuildCache.sh | 4 ++-- util/weekly_build/ShellSetup.sh | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index c0928ac84..a5bb8cc70 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -21,11 +21,12 @@ INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - if [ -z "$PACKAGES_TO_INSTALL" ]; then - spack fetch --missing 2>&1 | tee log.fetch + if [ "${SOURCE_CACHE::7}" == "file://" ]; then + mirrorpath=${SOURCE_CACHE} else - spack fetch --missing --dependencies $PACKAGES_TO_INSTALL 2>&1 | tee log.fetch + mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") fi + spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch # Just install the packages we're testing (+dependencies): spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # Install the rest of the stack as usual: diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh index 5210bb514..4ce53cba7 100755 --- a/util/weekly_build/05_BuildCache.sh +++ b/util/weekly_build/05_BuildCache.sh @@ -13,6 +13,6 @@ set -x for compiler in $COMPILERS; do cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} spack env activate . - spack buildcache push --unsigned --force ${BUILD_CACHE:-local-binary} $PACKAGES_TO_INSTALL - spack buildcache rebuild-index ${BUILD_CACHE:-local-binary} + spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL + spack buildcache rebuild-index ${BUILD_CACHE} done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index ff9ba802e..31e7f2e8e 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -23,4 +23,8 @@ function spack_install_exe { . $(dirname $0)/sites/${PLATFORM}.sh -echo "Build cache target name/directory: ${BUILD_CACHE?'BUILD_CACHE must be set (name of configured mirror or directory path!'}" +SOURCE_CACHE=${SOURCE_CACHE:-local-source} +BUILD_CACHE=${BUILD_CACHE:-local-binary} + +echo "Source cache mirror name/directory: ${SOURCE_CACHE}" +echo "Build cache mirror name/directory: ${BUILD_CACHE}" From e0576ca5139c54de2ef65d52a99938b1ebffb9eb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 30 Sep 2024 16:07:52 -0600 Subject: [PATCH 075/134] Update doc/source/PreConfiguredSites.rst: add note that oneAPI Narwhal setup currently does not compile Python packages --- doc/source/PreConfiguredSites.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 0ea8e3a7b..09ddec8d1 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -171,6 +171,8 @@ The following is required for building new spack environments with Intel on this The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! +.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack. + .. code-block:: console umask 0022 From 76361b06745500f2b1521589ec79ff291a7a5964 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 3 Oct 2024 15:46:18 -0600 Subject: [PATCH 076/134] Add configs/sites/tier1/atlantis/mirrors.yaml --- configs/sites/tier1/atlantis/mirrors.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 configs/sites/tier1/atlantis/mirrors.yaml diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml new file mode 100644 index 000000000..94e16cfaf --- /dev/null +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -0,0 +1,20 @@ +mirrors: + local-source: + fetch: + # TODO UPDATE? + url: file:///neptune_diagnostics/spack-stack/source-cache/ + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + # TODO UPDATE? + url: file:///neptune_diagnostics/spack-stack/source-cache/ + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null From 87067e943e5b51167ae139845f171b005bd31577 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Thu, 3 Oct 2024 22:45:20 -0400 Subject: [PATCH 077/134] Update site/packages.yaml and site/modules.yaml (#1322) Latest changes in site/packages.yaml and site/modules.yaml config files regarding ecFlow and QT. Removed exclude for creating ecflow modulefile, added path for qt and removed ecflow from package list. --- configs/sites/tier1/derecho/modules.yaml | 2 -- configs/sites/tier1/gaea-c5/modules.yaml | 2 -- configs/sites/tier1/gaea-c5/packages.yaml | 8 +------- configs/sites/tier1/gaea-c6/packages.yaml | 7 +------ configs/sites/tier1/hera/modules.yaml | 2 -- configs/sites/tier1/hera/packages.yaml | 11 ++++------- configs/sites/tier1/hercules/modules.yaml | 2 -- configs/sites/tier1/hercules/packages.yaml | 13 +++---------- configs/sites/tier1/jet/modules.yaml | 2 -- configs/sites/tier1/jet/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-aws/modules.yaml | 2 -- configs/sites/tier1/noaa-aws/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-azure/modules.yaml | 2 -- configs/sites/tier1/noaa-azure/packages.yaml | 12 +++++------- configs/sites/tier1/noaa-gcloud/modules.yaml | 2 -- configs/sites/tier1/noaa-gcloud/packages.yaml | 12 +++++------- configs/sites/tier1/orion/modules.yaml | 2 -- configs/sites/tier1/orion/packages.yaml | 7 ------- 18 files changed, 29 insertions(+), 83 deletions(-) diff --git a/configs/sites/tier1/derecho/modules.yaml b/configs/sites/tier1/derecho/modules.yaml index b9d5b4703..960198ae7 100644 --- a/configs/sites/tier1/derecho/modules.yaml +++ b/configs/sites/tier1/derecho/modules.yaml @@ -3,8 +3,6 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - cray-mpich - python diff --git a/configs/sites/tier1/gaea-c5/modules.yaml b/configs/sites/tier1/gaea-c5/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/gaea-c5/modules.yaml +++ b/configs/sites/tier1/gaea-c5/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index 6e13f710e..e9f4fc586 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -81,12 +81,6 @@ packages: externals: - spec: dos2unix@7.4.0 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4 - modules: [ecflow/5.8.4] file: externals: - spec: file@5.32 @@ -197,7 +191,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2/5.15.2/gcc_64 + prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 rdma-core: externals: - spec: rdma-core@37.0 diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 75473846c..7e9663d31 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -70,11 +70,6 @@ packages: externals: - spec: dos2unix@7.4.0 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/ecflow-5.8.4 file: externals: - spec: file@5.32 @@ -184,7 +179,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /autofs/ncrc-svm1_proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 + prefix: /ncrc/proj/epic/spack-stack/qt-5.15.2/5.15.2/gcc_64 rdma-core: externals: - spec: rdma-core@42.0 diff --git a/configs/sites/tier1/hera/modules.yaml b/configs/sites/tier1/hera/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/hera/modules.yaml +++ b/configs/sites/tier1/hera/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/hera/packages.yaml b/configs/sites/tier1/hera/packages.yaml index 894af0f5e..36a0a3f68 100644 --- a/configs/sites/tier1/hera/packages.yaml +++ b/configs/sites/tier1/hera/packages.yaml @@ -34,13 +34,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.5.3+ui+static_boost - prefix: /apps/ecflow/5.5.3 - modules: - - ecflow/5.5.3 file: externals: - spec: file@5.11 @@ -120,6 +113,10 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr + qt: + externals: + - spec: qt@5.12.6 + prefix: /scratch1/NCEPDEV/nems/role.epic/installs/qt-5.12.6/qtbase rsync: externals: - spec: rsync@3.1.2 diff --git a/configs/sites/tier1/hercules/modules.yaml b/configs/sites/tier1/hercules/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/hercules/modules.yaml +++ b/configs/sites/tier1/hercules/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml index 9c2c95209..0128505f1 100644 --- a/configs/sites/tier1/hercules/packages.yaml +++ b/configs/sites/tier1/hercules/packages.yaml @@ -19,13 +19,6 @@ packages: externals: - spec: diffutils@3.7 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 @@ -73,10 +66,10 @@ packages: prefix: /usr qt: externals: - - spec: qt@5.15.8 - prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd + - spec: qt@5.15.14 + prefix: /apps/contrib/spack-stack-1.1/gcc-11.3.1/qt-5.15.14-mfeuvcidmyqoi2m5i2tfrk6yd7xtk6pt modules: - - qt/5.15.8 + - qt/5.15.14 sed: externals: - spec: sed@4.8 diff --git a/configs/sites/tier1/jet/modules.yaml b/configs/sites/tier1/jet/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/jet/modules.yaml +++ b/configs/sites/tier1/jet/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml index 04633817d..fa6a0d123 100644 --- a/configs/sites/tier1/jet/packages.yaml +++ b/configs/sites/tier1/jet/packages.yaml @@ -34,13 +34,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.5.3+ui+static_boost - prefix: /apps/ecflow/5.5.3 - modules: - - ecflow/5.5.3 file: externals: - spec: file@5.11 @@ -124,6 +117,11 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.12.6 + prefix: /contrib/spack-stack/installs/qt-5.12.6/qtbase rsync: externals: - spec: rsync@3.1.2 diff --git a/configs/sites/tier1/noaa-aws/modules.yaml b/configs/sites/tier1/noaa-aws/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-aws/modules.yaml +++ b/configs/sites/tier1/noaa-aws/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml index ee153e8a5..dda1f369b 100644 --- a/configs/sites/tier1/noaa-aws/packages.yaml +++ b/configs/sites/tier1/noaa-aws/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -82,6 +75,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/noaa-azure/modules.yaml b/configs/sites/tier1/noaa-azure/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-azure/modules.yaml +++ b/configs/sites/tier1/noaa-azure/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml index 004aea213..397bc780b 100644 --- a/configs/sites/tier1/noaa-azure/packages.yaml +++ b/configs/sites/tier1/noaa-azure/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -78,6 +71,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/noaa-gcloud/modules.yaml b/configs/sites/tier1/noaa-gcloud/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/noaa-gcloud/modules.yaml +++ b/configs/sites/tier1/noaa-gcloud/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml index 004aea213..397bc780b 100644 --- a/configs/sites/tier1/noaa-gcloud/packages.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages.yaml @@ -15,13 +15,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.33 @@ -78,6 +71,11 @@ packages: externals: - spec: pkg-config@1.4.2 prefix: /usr + qt: + buildable: false + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 rsync: externals: - spec: rsync@3.1.3 diff --git a/configs/sites/tier1/orion/modules.yaml b/configs/sites/tier1/orion/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/orion/modules.yaml +++ b/configs/sites/tier1/orion/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/orion/packages.yaml b/configs/sites/tier1/orion/packages.yaml index 3fcecb71c..bef7849a9 100644 --- a/configs/sites/tier1/orion/packages.yaml +++ b/configs/sites/tier1/orion/packages.yaml @@ -31,13 +31,6 @@ packages: externals: - spec: diffutils@3.7 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/epic/role-epic/spack-stack/orion/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 From 91fe139a3d8fce9393f84d9435a0732266ae1ed2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 05:29:09 -0600 Subject: [PATCH 078/134] Add configs/sites/tier1/atlantis/packages_gcc.yaml --- .../sites/tier1/atlantis/packages_gcc.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 configs/sites/tier1/atlantis/packages_gcc.yaml diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml new file mode 100644 index 000000000..7d36ed0b4 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -0,0 +1,23 @@ +packages: + all: + compiler:: [gcc@11.2.0] + providers: + mpi:: [openmpi@4.1.6] + #blas:: [intel-oneapi-mkl] + #fftw-api:: [intel-oneapi-mkl] + #lapack:: [intel-oneapi-mkl] + slurm: + buildable: False + externals: + - spec: slurm@21.08.8 +pmix + modules: + - slurm/slurm/21.08.8 + openmpi: + require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' +modules: + default: + lmod: + include: + - openmpi + exclude: + - slurm From 5469c560ce72ac2d27317389a00a04b3d53a9c1a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 05:37:30 -0600 Subject: [PATCH 079/134] Remove site sandy --- configs/sites/tier1/sandy/compilers.yaml | 14 --- configs/sites/tier1/sandy/config.yaml | 2 - configs/sites/tier1/sandy/mirrors.yaml | 18 ---- configs/sites/tier1/sandy/modules.yaml | 8 -- configs/sites/tier1/sandy/packages.yaml | 113 ----------------------- 5 files changed, 155 deletions(-) delete mode 100644 configs/sites/tier1/sandy/compilers.yaml delete mode 100644 configs/sites/tier1/sandy/config.yaml delete mode 100644 configs/sites/tier1/sandy/mirrors.yaml delete mode 100644 configs/sites/tier1/sandy/modules.yaml delete mode 100644 configs/sites/tier1/sandy/packages.yaml diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml deleted file mode 100644 index b86ff0a33..000000000 --- a/configs/sites/tier1/sandy/compilers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -compilers: -- compiler: - spec: gcc@=9.3.0 - paths: - cc: /software/depot/gcc-9.3.0/bin/gcc - cxx: /software/depot/gcc-9.3.0/bin/g++ - f77: /software/depot/gcc-9.3.0/bin/gfortran - fc: /software/depot/gcc-9.3.0/bin/gfortran - flags: {} - operating_system: centos7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml deleted file mode 100644 index 84fe05ab9..000000000 --- a/configs/sites/tier1/sandy/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 8 diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/sandy/mirrors.yaml deleted file mode 100644 index c48f33bf1..000000000 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ /dev/null @@ -1,18 +0,0 @@ -mirrors: - local-source: - fetch: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml deleted file mode 100644 index aeb254c2e..000000000 --- a/configs/sites/tier1/sandy/modules.yaml +++ /dev/null @@ -1,8 +0,0 @@ -modules: - default: - enable:: - - tcl - tcl: - include: - # List of packages for which we need modules that are blacklisted by default - - python diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml deleted file mode 100644 index 1af1ab029..000000000 --- a/configs/sites/tier1/sandy/packages.yaml +++ /dev/null @@ -1,113 +0,0 @@ -packages: - all: - compiler:: [gcc@9.3.0] - providers: - mpi:: [openmpi@4.0.5] - -### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.0.5%gcc@9.3.0 - prefix: /software7/depot/openmpi-4.0.5 - -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - binutils: - externals: - - spec: binutils@2.30.117 - prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - curl: - externals: - - spec: curl@7.61.1 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - git: - externals: - - spec: git@2.39.2+tcltk - prefix: /software8/depot/git-2.39.2 - git-lfs: - externals: - - spec: git-lfs@3.3.0 - prefix: /software8/depot/git-2.39.2 - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - # Don't use external openssl, too old - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr - # Can't use with py-xnrl - #perl: - # externals: - # - spec: perl@5.26.3~cpanm+shared+threads - # prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - subversion: - externals: - - spec: subversion@1.10.2 - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr - texinfo: - externals: - - spec: texinfo@6.5 - prefix: /usr - wget: - externals: - - spec: wget@1.19.5 - prefix: /usr From ab722bb3cfa31a761aa78f25c6c2800e363467b1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 4 Oct 2024 10:21:35 -0600 Subject: [PATCH 080/134] More updates to atlantis site config for GNU --- configs/sites/tier1/atlantis/modules.yaml | 3 +++ configs/sites/tier1/atlantis/packages_gcc.yaml | 12 +++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index 11c5514e2..a3a386a9c 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,3 +6,6 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python + - openmpi + exclude: + - slurm diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml index 7d36ed0b4..c5e189964 100644 --- a/configs/sites/tier1/atlantis/packages_gcc.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -10,14 +10,8 @@ packages: buildable: False externals: - spec: slurm@21.08.8 +pmix - modules: - - slurm/slurm/21.08.8 + prefix: /cm/shared/apps/slurm/21.08.8 + #modules: + #- slurm/slurm/21.08.8 openmpi: require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' -modules: - default: - lmod: - include: - - openmpi - exclude: - - slurm From 46234a70f0c72f5e94c177773219eb8977a3ba9f Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:14:23 -0400 Subject: [PATCH 081/134] fix openmpi version on Hercules (#1326) --- configs/sites/tier1/hercules/packages_gcc.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/sites/tier1/hercules/packages_gcc.yaml b/configs/sites/tier1/hercules/packages_gcc.yaml index 5781224d8..37c496c45 100644 --- a/configs/sites/tier1/hercules/packages_gcc.yaml +++ b/configs/sites/tier1/hercules/packages_gcc.yaml @@ -2,13 +2,13 @@ packages: all: compiler:: [gcc@12.2.0] providers: - mpi:: [openmpi@4.1.6] + mpi:: [openmpi@4.1.4] mpi: buildable: False openmpi: externals: - - spec: openmpi@4.1.6%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath + - spec: openmpi@4.1.4%gcc@12.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath schedulers=slurm modules: - gcc/12.2.0 - - openmpi/4.1.6-gcc-12.2.0-spack + - openmpi/4.1.4 From 5101d55845ad79c591874367ec34262d80b22229 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 7 Oct 2024 15:50:04 -0600 Subject: [PATCH 082/134] For release/1.8.0: update S4 site config and documentation (#1328) * Update site config and documentation for S4 after Rocky8 update (Intel only so far with gcc@13 backend) --- configs/sites/tier1/s4/compilers.yaml | 48 +++++---- configs/sites/tier1/s4/packages.yaml | 141 +++++++++++++------------- doc/source/PreConfiguredSites.rst | 5 - 3 files changed, 102 insertions(+), 92 deletions(-) diff --git a/configs/sites/tier1/s4/compilers.yaml b/configs/sites/tier1/s4/compilers.yaml index d364dd768..dff2729fb 100644 --- a/configs/sites/tier1/s4/compilers.yaml +++ b/configs/sites/tier1/s4/compilers.yaml @@ -1,34 +1,46 @@ compilers: - compiler: - spec: intel@2021.5.0 + spec: intel@=2021.10.0 paths: - cc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icc - cxx: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/icpc - f77: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort - fc: /opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/bin/intel64/ifort + cc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/bin/intel64/ifort flags: {} - operating_system: centos7 + operating_system: rocky8 target: x86_64 modules: - - license_intel/S4 - - intel/2022.1 + - intel/2023.2 environment: prepend_path: - PATH: '/data/prod/hpc-stack/gnu/9.3.0/bin' - LD_LIBRARY_PATH: '/home/opt/intel/oneapi/2022.1/compiler/2022.0.1/linux/compiler/lib/intel64_lin:/data/prod/hpc-stack/gnu/9.3.0/lib64' - CPATH: '/data/prod/hpc-stack/gnu/9.3.0/include' + PATH: /opt/gcc/13.3/bin + LD_LIBRARY_PATH: /opt/intel/oneapi/2023.2/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/opt/gcc/13.3/lib64:/opt/gcc/13.3/lib + CPATH: /opt/gcc/13.3/include extra_rpaths: [] - compiler: - spec: gcc@9.3.0 + spec: gcc@=13.3.0 paths: - cc: /data/prod/hpc-stack/gnu/9.3.0/bin/gcc - cxx: /data/prod/hpc-stack/gnu/9.3.0/bin/g++ - f77: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran - fc: /data/prod/hpc-stack/gnu/9.3.0/bin/gfortran + cc: /opt/gcc/13.3/bin/gcc + cxx: /opt/gcc/13.3/bin/g++ + f77: /opt/gcc/13.3/bin/gfortran + fc: /opt/gcc/13.3/bin/gfortran flags: {} - operating_system: centos7 + operating_system: rocky8 target: x86_64 modules: - - gnu/9.3.0 + - gcc/13.3 + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index e0312b799..a2457222e 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -1,10 +1,8 @@ packages: all: - compiler:: [intel@2021.5.0,gcc@9.3.0] + compiler:: [intel@2021.10.0,gcc@13.3.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.0] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -15,15 +13,18 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0 - prefix: /opt/intel/oneapi/2022.1 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + prefix: /opt/intel/oneapi/2023.2 + modules: + - intel/2023.2 intel-oneapi-mkl: # Remove buildable: False and uncomment externals section below to use intel-oneapi-mkl buildable: False #externals: - #- spec: intel-oneapi-mkl@2022.0.1%intel@2021.5.0 - # prefix: /opt/intel/oneapi/2022.1 - + #- spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0 + # prefix: /opt/intel/oneapi/2023.2 + # modules: + # - intel/2023.2 # Remove this section to switch to intel-oneapi-mkl ectrans: require:: @@ -42,47 +43,63 @@ packages: prefix: /usr automake: externals: - - spec: automake@1.15 + - spec: automake@1.16.1 prefix: /usr bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 + prefix: /usr + binutils: + externals: + - spec: binutils@2.30.123 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 prefix: /usr + cmake: + externals: + - spec: cmake@3.26.5 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr - diffutils: + curl: externals: - - spec: diffutils@3.3 + - spec: curl@7.61.1+gssapi+ldap+nghttp2 prefix: /usr - doxygen: + diffutils: externals: - - spec: doxygen@1.8.5+graphviz~mscgen + - spec: diffutils@3.6 prefix: /usr file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr flex: externals: - - spec: flex@2.5.37+lex + - spec: flex@2.6.1+lex prefix: /usr gawk: externals: - - spec: gawk@4.0.2 + - spec: gawk@4.2.1 prefix: /usr gettext: externals: @@ -90,105 +107,91 @@ packages: prefix: /usr ghostscript: externals: - - spec: ghostscript@9.25 + - spec: ghostscript@9.27 prefix: /usr git: externals: - - spec: git@2.30.0+tcltk - prefix: /opt/git/2.30.0 - modules: - - git/2.30.0 + - spec: git@2.43.5+tcltk + prefix: /usr git-lfs: externals: - - spec: git-lfs@2.10.0 + - spec: git-lfs@3.5.1 prefix: /usr gmake: externals: - - spec: gmake@3.82 + - spec: gmake@4.2.1 prefix: /usr go: externals: - - spec: go@1.11.5 + - spec: go@1.21.13 prefix: /usr groff: externals: - - spec: groff@1.22.2 - prefix: /usr - openjdk: - externals: - - spec: openjdk@1.8.0_262-b10 + - spec: groff@1.22.3 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - libfuse: - externals: - - spec: libfuse@2.9.2 + - spec: krb5@1.18.2 prefix: /usr libtool: externals: - - spec: libtool@2.4.2 + - spec: libtool@2.4.6 prefix: /usr - libxpm: + m4: externals: - - spec: libxpm@4.11.0 + - spec: m4@1.4.18 prefix: /usr - lustre: + openssh: externals: - - spec: lustre@2.10.5 + - spec: openssh@8.0p1 prefix: /usr - m4: + openssl: externals: - - spec: m4@1.4.16 + - spec: openssl@1.1.1k prefix: /usr - mysql: - buildable: False - externals: - - spec: mysql@8.0.31 - prefix: /data/prod/jedi/spack-stack/mysql-8.0.31 - modules: - - mysql/8.0.31 - pkg-config: + perl: externals: - - spec: pkg-config@0.27.1 + - spec: perl@5.26.3~cpanm+opcode+open+shared+threads prefix: /usr - qt: - buildable: False + pkgconf: externals: - - spec: qt@5.9.7 + - spec: pkgconf@1.4.2 prefix: /usr + # TODO - installing via spack-stack works, for example for + # spack-stack-1.8.0, but would be good if we didn't have to. + # https://github.com/JCSDA/spack-stack/issues/1329 + #qt: + # buildable: False + # externals: + # - spec: qt@5.9.7 + # prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 - prefix: /usr - texlive: - externals: - - spec: texlive@20130530 + - spec: texinfo@6.5 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr xz: externals: - - spec: xz@5.2.2 + - spec: xz@5.2.4 prefix: /usr zip: externals: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 2ea670ac2..bfcd808d3 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -343,14 +343,9 @@ UW (Univ. of Wisconsin) S4 The following is required for building new spack environments with any supported compiler on this platform. -**NEEDS UPDATING** - .. code-block:: console module purge - module use /data/prod/jedi/spack-stack/modulefiles - module load miniconda/3.9.12 - module load ecflow/5.8.4 .. _Preconfigured_Sites_AWS_Parallelcluster: From cbf2bd3f66da87278b97b45af974d2a9339e4c1e Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:55:36 -0400 Subject: [PATCH 083/134] Release 1.8.0 additional fixes for Jet, Hercules and Orion (after running ufs-weather-model) (#1331) * Update Jet packages_intel.yaml * Update Orion packages_intel.yaml * Update Orion packages_gcc.yaml * Update Hercules packages.yaml --- configs/sites/tier1/hercules/packages.yaml | 12 ++++++++++++ configs/sites/tier1/jet/packages_intel.yaml | 2 ++ configs/sites/tier1/orion/packages_gcc.yaml | 2 -- configs/sites/tier1/orion/packages_intel.yaml | 2 -- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/hercules/packages.yaml b/configs/sites/tier1/hercules/packages.yaml index 0128505f1..bb6e6222c 100644 --- a/configs/sites/tier1/hercules/packages.yaml +++ b/configs/sites/tier1/hercules/packages.yaml @@ -15,6 +15,10 @@ packages: externals: - spec: coreutils@8.32 prefix: /usr + curl: + externals: + - spec: curl@7.76.1+gssapi+ldap+nghttp2 + prefix: /usr diffutils: externals: - spec: diffutils@3.7 @@ -60,6 +64,14 @@ packages: prefix: /work/noaa/epic/role-epic/spack-stack/hercules/mysql-8.0.31 modules: - mysql/8.0.31 + openssh: + externals: + - spec: openssh@8.7p1 + prefix: /usr + openssl: + externals: + - spec: openssl@3.0.1 + prefix: /usr pkgconf: externals: - spec: pkgconf@1.7.3 diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index cc0cac045..5e0b82aaf 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -8,6 +8,8 @@ packages: intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + modules: + - impi/2022.1.2 prefix: /apps/oneapi intel-oneapi-mkl: externals: diff --git a/configs/sites/tier1/orion/packages_gcc.yaml b/configs/sites/tier1/orion/packages_gcc.yaml index 8a8f8d641..aadfeb18c 100644 --- a/configs/sites/tier1/orion/packages_gcc.yaml +++ b/configs/sites/tier1/orion/packages_gcc.yaml @@ -3,8 +3,6 @@ packages: compiler:: [gcc@12.2.0] providers: mpi:: [openmpi@4.1.4] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] mpi: buildable: False openmpi: diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 8c1e33385..eb4a22dc7 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -3,8 +3,6 @@ packages: compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] - # https://github.com/JCSDA/spack-stack/issues/1055 - zlib-api:: [zlib] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] From 9d6416c1fc2af0fb4616a1940798763eb3d1cb35 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 8 Oct 2024 10:09:21 -0700 Subject: [PATCH 084/134] go back to spack-stack-dev branch --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 5c7383f2a..a69f3e4f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "spack"] path = spack url = https://github.com/jcsda/spack - branch = release/1.8.0 + branch = spack-stack-dev [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules From 5068f19444eaa1564c66a6e9ff751bf7fac16be6 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Wed, 9 Oct 2024 22:00:16 +0000 Subject: [PATCH 085/134] use shorter padded length on acorn because of intel19 segfault w/esmf --- util/weekly_build/sites/acorn.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index 8fb170d17..ef19a4962 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -15,10 +15,11 @@ function spack_install_exe { shift 1 ${SPACK_STACK_DIR}/util/parallel_install.sh 3 4 $* } -PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" -PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" function alert_cmd { module purge # annoying libstdc++ issue mail -s 'spack-stack weekly build failure' alexander.richert@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") } +PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" +PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" +PADDED_LENGTH=140 TEST_UFSWM=ON From 4ecbdae943a6709d3221ad13bf3a8a1ba50ded71 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 11 Oct 2024 15:09:42 -0600 Subject: [PATCH 086/134] Next round of updates for Atlantis: use Bright OpenMPI --- configs/sites/tier1/atlantis/modules.yaml | 3 --- configs/sites/tier1/atlantis/packages_gcc.yaml | 18 +++++++----------- .../sites/tier1/atlantis/packages_intel.yaml | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index a3a386a9c..11c5514e2 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,6 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - - openmpi - exclude: - - slurm diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml index c5e189964..bc10f8b75 100644 --- a/configs/sites/tier1/atlantis/packages_gcc.yaml +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -2,16 +2,12 @@ packages: all: compiler:: [gcc@11.2.0] providers: - mpi:: [openmpi@4.1.6] - #blas:: [intel-oneapi-mkl] - #fftw-api:: [intel-oneapi-mkl] - #lapack:: [intel-oneapi-mkl] - slurm: + mpi:: [openmpi@4.1.5] + mpi: buildable: False - externals: - - spec: slurm@21.08.8 +pmix - prefix: /cm/shared/apps/slurm/21.08.8 - #modules: - #- slurm/slurm/21.08.8 openmpi: - require: '@4.1.6 +gpfs +romio fabrics=ucx romio-filesystem=gpfs schedulers=slurm +pmi +internal-pmix +internal-hwloc +internal-libevent +legacylaunchers' + buildable: False + externals: + - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + modules: + - openmpi/mlnx/gcc/64/4.1.5a1 diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index ea2f4de32..4271f3f10 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -15,6 +15,6 @@ packages: intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.1.0 + prefix: /cm/shared/apps/intel/oneapi modules: - mkl/2022.1.0 - prefix: /cm/shared/apps/intel/oneapi From d6a41f55b18961db57997ef8bec38b1142b943db Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 11 Oct 2024 15:10:38 -0600 Subject: [PATCH 087/134] Update configs/common/packages.yaml: do not require, but prefer, the Python variant for ESMF, set accordingly in spack-ext/repos/spack-stack/packages/neptune-env/package.py --- configs/common/packages.yaml | 24 ++++++++++--------- .../packages/neptune-env/package.py | 3 ++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index fc7089741..1b48f823b 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -54,17 +54,19 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio +python' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - #- any_of: [''] - # when: "%gcc" - # message: "Extra ESMF compile options for GCC" - #- any_of: [''] - # when: "%apple-clang" - # message: "Extra ESMF compile options for GCC" + - '~xerces ~pnetcdf +shared +external-parallelio' + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: "%intel" + message: "Extra ESMF compile options for Intel" + #- any_of: [''] + # when: "%gcc" + # message: "Extra ESMF compile options for GCC" + #- any_of: [''] + # when: "%apple-clang" + # message: "Extra ESMF compile options for GCC" + prefer: + - '+python' fckit: require: '@0.11.0 +eckit' fftw: diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index f79968eac..94c7baac7 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -39,7 +39,8 @@ class NeptuneEnv(BundlePackage): depends_on("ip@5:", type="run", when="%gcc") depends_on("ip@5:", type="run", when="%intel") depends_on("ip@5:", type="run", when="%oneapi") - depends_on("esmf", type="run") + depends_on("esmf +python", when="+python", type="run") + depends_on("esmf ~python", when="~python", type="run") depends_on("nco", type="run") depends_on("mct", type="run") From 28ea3f72cf0dbd1c15e0dba67fd3b3dac1ff32da Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:00:50 -0600 Subject: [PATCH 088/134] Remove legacy environment variables for hdf5, p4est, libyaml from configs/common/modules_{lmod,tcl}.yaml --- configs/common/modules_lmod.yaml | 16 ---------------- configs/common/modules_tcl.yaml | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 41bfa24df..7c994ce45 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -112,20 +112,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -151,12 +141,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 5ce83e8b4..551c12e65 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -114,20 +114,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -153,12 +143,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: From cd7c3208a04fc42339f03a4e7f004de4b47510ff Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:02:09 -0600 Subject: [PATCH 089/134] Split neptune-env into neptune-env (with default espc false) and neptune-python-env --- configs/templates/neptune-dev/spack.yaml | 3 +- .../packages/neptune-env/package.py | 33 ++++----------- .../packages/neptune-python-env/package.py | 41 +++++++++++++++++++ 3 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 spack-ext/repos/spack-stack/packages/neptune-python-env/package.py diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 5f3332f79..e0219fa0d 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,7 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc +python +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 + - neptune-python-env +xnrl specs: - matrix: diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index f79968eac..ddd732692 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,7 +9,7 @@ class NeptuneEnv(BundlePackage): - """Development environment for neptune standalone""" + """Development environment for NEPTUNE standalone""" # Fake URL homepage = "https://github.com/notavalidaccount/neptune" @@ -17,11 +17,9 @@ class NeptuneEnv(BundlePackage): maintainers("climbfuji", "areinecke") - version("1.4.0") + version("1.5.0") - variant("python", default=True, description="Build Python dependencies") - variant("espc", default=True, description="Build ESPC dependencies") - variant("xnrl", default=True, description="Build XNRL and its extra Python dependencies") + variant("espc", default=False, description="Build ESPC dependencies") depends_on("base-env", type="run") @@ -33,7 +31,6 @@ class NeptuneEnv(BundlePackage): depends_on("libyaml", type="run") depends_on("p4est", type="run") depends_on("w3emc", type="run") - depends_on("w3nco", type="run") depends_on("sp", type="run", when="%aocc") depends_on("ip@5:", type="run", when="%apple-clang") depends_on("ip@5:", type="run", when="%gcc") @@ -43,28 +40,12 @@ class NeptuneEnv(BundlePackage): depends_on("nco", type="run") depends_on("mct", type="run") - conflicts("+xnrl", when="~python", msg="Variant xnrl requires variant python") - with when("+espc"): depends_on("fftw", type="build") depends_on("netlib-lapack", type="build") - with when("+python"): - depends_on("py-f90nml", type="run") - depends_on("py-h5py", type="run") - depends_on("py-netcdf4", type="run") - depends_on("py-pandas", type="run") - depends_on("py-pycodestyle", type="run") - depends_on("py-pybind11", type="run") - depends_on("py-pyhdf", type="run") - depends_on("py-python-dateutil", type="run") - depends_on("py-pyyaml", type="run") - depends_on("py-scipy", type="run") - depends_on("py-xarray", type="run") - depends_on("py-pytest", type="run") - depends_on("py-fortranformat", type="run") - - with when("+xnrl"): - depends_on("py-xnrl", type="run") + # Basic Python dependencies that are always needed + depends_on("py-f90nml", type="run") + depends_on("py-python-dateutil", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py new file mode 100644 index 000000000..5b6f243b8 --- /dev/null +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -0,0 +1,41 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys + +from spack.package import * + + +class NeptunePythonEnv(BundlePackage): + """Development environment for NEPTUNE standalone with all Python dependencies""" + + # Fake URL + homepage = "https://github.com/notavalidaccount/neptune" + git = "https://github.com/notavalidaccount/neptune.git" + + maintainers("climbfuji", "areinecke") + + version("1.5.0") + + variant("xnrl", default=False, description="Build non-pulic XNRL") + + depends_on("neptune-env", type="run") + + depends_on("py-h5py", type="run") + depends_on("py-netcdf4", type="run") + depends_on("py-pandas", type="run") + depends_on("py-pycodestyle", type="run") + depends_on("py-pybind11", type="run") + depends_on("py-pyhdf", type="run") + depends_on("py-pyyaml", type="run") + depends_on("py-scipy", type="run") + depends_on("py-xarray", type="run") + depends_on("py-pytest", type="run") + depends_on("py-fortranformat", type="run") + + with when("+xnrl"): + depends_on("py-xnrl", type="run") + + # There is no need for install() since there is no code. From 886cf903d9116e2b05a47240a05efcde0e47b193 Mon Sep 17 00:00:00 2001 From: Nate Crossette Date: Mon, 14 Oct 2024 06:22:32 -0600 Subject: [PATCH 090/134] Add note to docs for selecting branch when cloning (#1341) --- doc/source/NewSiteConfigs.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 036fa41f9..65b59b6b0 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -211,11 +211,11 @@ Creating a new environment Remember to activate the ``lua`` module environment and have MacTeX in your search path, if applicable. It is also recommended to increase the stacksize limit to 65Kb using ``ulimit -S -s unlimited``. -1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. +1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} @@ -500,11 +500,11 @@ Creating a new environment It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimited``, and to test if the module environment functions correctly (``module available``). -1. You will need to clone spack-stack and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. +1. You will need to clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. It is also a good idea to save the directory in your environment for later use. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} @@ -675,11 +675,11 @@ With all of that in mind, the following instructions were used on an Amazon Web module use /opt/nvidia/hpc_sdk/modulefiles module load nvhpc-openmpi3/24.3 -4. Clone spack-stack and its dependencies and activate the spack-stack tool. +4. Clone spack-stack (selecting your desired spack-stack branch) and its dependencies and activate the spack-stack tool. .. code-block:: console - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git + git clone [-b develop OR release/branch-name] --recurse-submodules https://github.com/jcsda/spack-stack.git cd spack-stack # Sources Spack from submodule and sets ${SPACK_STACK_DIR} From 3d25bfb129406711d6a1067602116eb65dc9570b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 14 Oct 2024 10:55:30 -0600 Subject: [PATCH 091/134] Update configs/templates/{neptune-dev,skylab-dev,unified-dev} --- configs/templates/neptune-dev/spack.yaml | 4 ++-- configs/templates/skylab-dev/spack.yaml | 3 ++- configs/templates/unified-dev/spack.yaml | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index e0219fa0d..564cf6d86 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl + - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 + - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 1ba272a63..4616138ca 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -17,7 +17,8 @@ spack: - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env # Various crtm tags (list all to avoid duplicate packages) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index a05ed4a75..608b4462a 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -21,7 +21,8 @@ spack: - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 From d6641e4968207de1b83791fdd518e76660c488d8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 16 Oct 2024 10:40:34 -0600 Subject: [PATCH 092/134] Add py-pyyaml to neptune-env, remove invalid Python variant from neptune-env and add esmf+python to neptune-python-env --- spack-ext/repos/spack-stack/packages/neptune-env/package.py | 4 ++-- .../repos/spack-stack/packages/neptune-python-env/package.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index a8d3d5807..ff7aef9d4 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -36,8 +36,7 @@ class NeptuneEnv(BundlePackage): depends_on("ip@5:", type="run", when="%gcc") depends_on("ip@5:", type="run", when="%intel") depends_on("ip@5:", type="run", when="%oneapi") - depends_on("esmf +python", when="+python", type="run") - depends_on("esmf ~python", when="~python", type="run") + depends_on("esmf", type="run") depends_on("nco", type="run") depends_on("mct", type="run") @@ -48,5 +47,6 @@ class NeptuneEnv(BundlePackage): # Basic Python dependencies that are always needed depends_on("py-f90nml", type="run") depends_on("py-python-dateutil", type="run") + depends_on("py-pyyaml", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py index 5b6f243b8..2e20105b8 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") depends_on("neptune-env", type="run") + # Enable the Python variant for ESMF + depends_on("esmf +python", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run") From c6a067f7730e4554fd75f9685d603d10f60c55de Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 16 Oct 2024 14:33:29 -0600 Subject: [PATCH 093/134] Update site config and documentation for Narwhal to use new, official oneapi@2024.2 installation --- configs/sites/tier1/narwhal/compilers.yaml | 16 ++++------------ .../sites/tier1/narwhal/packages_oneapi.yaml | 19 +++++++------------ doc/source/PreConfiguredSites.rst | 10 ++-------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 5e2e17c11..838a8a1dd 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -22,12 +22,8 @@ compilers:: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] - compiler: - spec: oneapi@2024.2.1 + spec: oneapi@2024.2.0 paths: - #cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx - #cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx - #f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort - #fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort cc: cc cxx: CC f77: ftn @@ -35,11 +31,8 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - tbb/2021.13 - - compiler-rt/2024.2.1 - - compiler/2024.2.1 - - ifort/2024.2.1 + - PrgEnv-intel/8.4.0 + - intel/2024.2 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: @@ -47,9 +40,8 @@ compilers:: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' - MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' + CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml index 9b032dcf1..4724e25a3 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -7,26 +7,21 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers + - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - mkl/2024.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 intel-oneapi-tbb: externals: - - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 modules: - tbb/2021.13 intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 - modules: - - tbb/2021.13 - - compiler-rt/2024.2.1 + - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 79567859e..cd3b8e3bb 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -166,19 +166,13 @@ The following is required for building new spack environments with Intel on this The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! -.. note:: This particular setup of the oneAPI compilers is outside the Cray-blessed environment and there are problems installing Python packages with spack. - .. code-block:: console umask 0022 module unload PrgEnv-cray - module load PrgEnv-intel/8.3.3 + module load PrgEnv-intel/8.4.0 module unload intel - module use /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles - module load tbb/2021.13 - module load compiler-rt/2024.2.1 - module load compiler/2024.2.1 - module load ifort/2024.2.1 + module load intel/2024.2 module unload cray-mpich module unload craype-network-ofi # Warning. Do not load craype-network-ucx From 716932c9dc5e4597c60fc61c50877e13694fa742 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 18 Oct 2024 07:59:46 -0600 Subject: [PATCH 094/134] Fix typo in configs/sites/tier1/narwhal/packages_oneapi.yaml --- configs/sites/tier1/narwhal/packages_oneapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml index 4724e25a3..3bcf99f38 100644 --- a/configs/sites/tier1/narwhal/packages_oneapi.yaml +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.2.1,gcc@10.3.0] + compiler:: [oneapi@2024.2.0,gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.21] mpi: From 41b59e35d7358e2418f1bbad04a2208928ef7be5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 18 Oct 2024 18:37:17 -0600 Subject: [PATCH 095/134] Update submodule pointer for spack (esmf@8.7.0) and replace esmf@8.7.0b11 with esmf@8.7.0 in all configs --- configs/common/modules_lmod.yaml | 2 ++ configs/common/modules_tcl.yaml | 2 ++ configs/common/packages | 0 configs/common/packages.yaml | 2 +- configs/templates/neptune-dev/spack.yaml | 4 ++-- configs/templates/skylab-dev/spack.yaml | 8 ++++---- configs/templates/unified-dev/spack.yaml | 8 ++++---- spack | 2 +- 8 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 configs/common/packages diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 7c994ce45..8fe637fb6 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,6 +136,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 551c12e65..3275aee44 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -138,6 +138,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/packages b/configs/common/packages new file mode 100644 index 000000000..e69de29bb diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 1b48f823b..99f17b060 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -55,7 +55,7 @@ packages: esmf: require: - '~xerces ~pnetcdf +shared +external-parallelio' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: "%intel" message: "Extra ESMF compile options for Intel" diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 564cf6d86..9a9e3dfec 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0 + - neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 4616138ca..be901ec7e 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,11 +14,11 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -27,7 +27,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 608b4462a..370a03e7c 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -17,12 +17,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 @@ -33,7 +33,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/spack b/spack index 6f9a6ba4e..b9f2dd225 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9 +Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702 From 4b2afed39127320af04f3b1d80d2e98dc17db101 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 09:44:20 -0600 Subject: [PATCH 096/134] Change spack source-cache locations on atlantis, narwhal, nautilus --- configs/sites/tier1/atlantis/mirrors.yaml | 2 -- configs/sites/tier1/narwhal/mirrors.yaml | 4 ++-- configs/sites/tier1/nautilus/mirrors.yaml | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml index 94e16cfaf..4d363e009 100644 --- a/configs/sites/tier1/atlantis/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,7 +1,6 @@ mirrors: local-source: fetch: - # TODO UPDATE? url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null @@ -10,7 +9,6 @@ mirrors: profile: null endpoint_url: null push: - # TODO UPDATE? url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null From 84d8f78692067cafd159fb153d0ff7bedc57e540 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 10:36:44 -0600 Subject: [PATCH 097/134] Add TEMPLATES list and loop over templates; rename util/weekly_build/03_SetupUnifiedEnv.sh to util/weekly_build/03_SetupEnv.sh --- util/weekly_build/03_SetupEnv.sh | 33 +++++++++++++++++++ util/weekly_build/03_SetupUnifiedEnv.sh | 32 ------------------ .../SpackStackBuildCache_AllSteps.sh | 3 +- util/weekly_build/sites/acorn.sh | 1 + util/weekly_build/sites/nautilus.sh | 2 +- 5 files changed, 36 insertions(+), 35 deletions(-) create mode 100755 util/weekly_build/03_SetupEnv.sh delete mode 100755 util/weekly_build/03_SetupUnifiedEnv.sh diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh new file mode 100755 index 000000000..40704086d --- /dev/null +++ b/util/weekly_build/03_SetupEnv.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +set -ex + +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + echo "Setting up environment build-${compiler/@/-} using template $template" + rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} + spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler + cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} + spack env activate . + spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" + # Check for duplicates and fail before doing the "real" concretization: + spack concretize --fresh 2>&1 | tee log.concretize + ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf + spack concretize --fresh --force +# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. +# spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test + # Get path to local-source mirror from spack site config + SOURCE_MIRROR=`spack mirror list | grep local-source` + SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} + # Download all source codes needed to build the environment + spack mirror create -a -d ${SOURCE_MIRROR} + done +done diff --git a/util/weekly_build/03_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupUnifiedEnv.sh deleted file mode 100755 index a6d360ed5..000000000 --- a/util/weekly_build/03_SetupUnifiedEnv.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi - -set -ex - -cd $RUNDIR/$RUNID - -set +x -. setup.sh -set -x - -for compiler in $COMPILERS; do - echo "Setting up environment build-${compiler/@/-} using template unified-dev" - #echo "Setting up environment build-${compiler/@/-} using template unified-dev" - rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" - # Check for duplicates and fail before doing the "real" concretization: - spack concretize --fresh 2>&1 | tee log.concretize - ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf - spack concretize --fresh --force -# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. -# spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test - # Get path to local-source mirror from spack site config - SOURCE_MIRROR=`spack mirror list | grep local-source` - SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} - # Download all source codes needed to build the environment - spack mirror create -a -d ${SOURCE_MIRROR} -done diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 0f8bb3870..73a7931c4 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -18,8 +18,7 @@ function trap_and_run { trap_and_run ./01_DirectorySetup.sh $* trap_and_run ./02_GetSpackStack.sh $* -#trap_and_run ./03_SetupEnv.sh $* -trap_and_run ./03_SetupUnifiedEnv.sh $* +trap_and_run ./03_SetupEnv.sh $* trap_and_run ./04_SpackInstall.sh $* trap_and_run ./05_BuildCache.sh $* trap_and_run ./06_AppTests.sh $* diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index ef19a4962..d84094f7b 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -1,5 +1,6 @@ module load gcc/11.2.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} +TEMPLATES=${TEMPLATES:-"unified-dev"} function spack_install_exe { # set +e # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh index 20c2497ea..916b799a7 100644 --- a/util/weekly_build/sites/nautilus.sh +++ b/util/weekly_build/sites/nautilus.sh @@ -1,6 +1,6 @@ COMPILERS=${COMPILERS:-"intel oneapi gcc"} +TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"} module purge umask 0022 SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack SPACK_STACK_BRANCH=feature/weekly_build_nautilus -### NEEDED ? BUILD_CACHE_DIR=${BUILD_CACHE_DIR:-/p/app/projects/NEPTUNE/spack-stack/build-cache} \ No newline at end of file From 0cdcb52bbc790d1b192c6ca272b8dd1372f5c096 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 10:38:59 -0600 Subject: [PATCH 098/134] Update names of weekly build scripts in Github actions --- .github/actions/Nightly_02_GetSpackStack/action.yaml | 11 +++++++++++ .../Nightly_02_GetSpackStackDevelop/action.yaml | 11 ----------- .../action.yaml | 4 ++-- .github/workflows/nautilus-nightly-intel.yaml | 8 ++++---- .github/workflows/nautilus-nightly-oneapi.yaml | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 .github/actions/Nightly_02_GetSpackStack/action.yaml delete mode 100644 .github/actions/Nightly_02_GetSpackStackDevelop/action.yaml rename .github/actions/{Nightly_03_SetupUnifiedEnv => Nightly_03_SetupEnv}/action.yaml (70%) diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml new file mode 100644 index 000000000..19a558bcb --- /dev/null +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 02 Get Spack-Stack + +runs: + using: "composite" + steps: + - name: Nightly 02 Get Spack-Stack + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./02_GetSpackStack.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml b/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml deleted file mode 100644 index 7565046dd..000000000 --- a/.github/actions/Nightly_02_GetSpackStackDevelop/action.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: Nightly 02 Get Spack-Stack Develop - -runs: - using: "composite" - steps: - - name: Nightly 02 Get Spack-Stack Develop - shell: bash - run: | - RUNID=`date +"%Y%m%d"` - cd util/weekly_build - ./02_GetSpackStackDevelop.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml b/.github/actions/Nightly_03_SetupEnv/action.yaml similarity index 70% rename from .github/actions/Nightly_03_SetupUnifiedEnv/action.yaml rename to .github/actions/Nightly_03_SetupEnv/action.yaml index 0f598cb88..48256bdc2 100644 --- a/.github/actions/Nightly_03_SetupUnifiedEnv/action.yaml +++ b/.github/actions/Nightly_03_SetupEnv/action.yaml @@ -1,9 +1,9 @@ -name: Nightly 03 Setup Unified Env +name: Nightly 03 Setup Env runs: using: "composite" steps: - - name: Nightly 03 Setup Unified Env + - name: Nightly 03 Setup Env shell: bash run: | RUNID=`date +"%Y%m%d"` diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml index 7dbf11bec..90767bb18 100644 --- a/.github/workflows/nautilus-nightly-intel.yaml +++ b/.github/workflows/nautilus-nightly-intel.yaml @@ -33,9 +33,9 @@ jobs: clean: true - name: 01_DirectorySetup uses: ./.github/actions/Nightly_01_DirectorySetup - - name: 02_GetSpackStackDevelop - uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - - name: 03_SetupUnifiedEnv - uses: ./.github/actions/Nightly_03_SetupUnifiedEnv + - name: 02_GetSpackStack + uses: ./.github/actions/Nightly_02_GetSpackStack + - name: 03_SetupEnv + uses: ./.github/actions/Nightly_03_SetupEnv - name: 04_SpackInstall uses: ./.github/actions/Nightly_04_SpackInstall diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index df7b708a5..cd8823cb0 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -33,9 +33,9 @@ jobs: clean: true - name: 01_DirectorySetup uses: ./.github/actions/Nightly_01_DirectorySetup - - name: 02_GetSpackStackDevelop - uses: ./.github/actions/Nightly_02_GetSpackStackDevelop - - name: 03_SetupUnifiedEnv - uses: ./.github/actions/Nightly_03_SetupUnifiedEnv + - name: 02_GetSpackStack + uses: ./.github/actions/Nightly_02_GetSpackStack + - name: 03_SetupEnv + uses: ./.github/actions/Nightly_03_SetupEnv - name: 04_SpackInstall uses: ./.github/actions/Nightly_04_SpackInstall From 30aa75825bf256b0137bc99e7c0af5f8c6cb8047 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 10:52:03 -0600 Subject: [PATCH 099/134] Set envname and envdir, only perform packages_to_test install step if variable defined --- util/weekly_build/03_SetupEnv.sh | 18 ++++++----- util/weekly_build/04_SpackInstall.sh | 48 +++++++++++----------------- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 40704086d..d9fe5a13c 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -12,10 +12,12 @@ set -x for compiler in $COMPILERS; do for template in $TEMPLATES; do - echo "Setting up environment build-${compiler/@/-} using template $template" - rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/build-$envname + echo "Setting up environment $envname in $envdir" + rm -rf $envdir spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} + cd $envdir spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: @@ -24,10 +26,10 @@ for compiler in $COMPILERS; do spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test - # Get path to local-source mirror from spack site config - SOURCE_MIRROR=`spack mirror list | grep local-source` - SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} - # Download all source codes needed to build the environment - spack mirror create -a -d ${SOURCE_MIRROR} +### # Get path to local-source mirror from spack site config +### SOURCE_MIRROR=`spack mirror list | grep local-source` +### SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} +### # Download all source codes needed to build the environment +### spack mirror create -a -d ${SOURCE_MIRROR} done done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index bcd2750ae..857d7f472 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -10,9 +10,6 @@ set +x . setup.sh set -x -<<<<<<< HEAD -INSTALL_OPTS="--show-log-on-error --fail-fast --no-cache $INSTALL_OPTS" -======= if [ "$REUSE_BUILD_CACHE" == YES ]; then cache_flag="--no-check-signature" else @@ -20,32 +17,25 @@ else fi INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" ->>>>>>> 5068f19444eaa1564c66a6e9ff751bf7fac16be6 for compiler in $COMPILERS; do - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . -<<<<<<< HEAD - #spack fetch # 2>&1 | tee log.fetch - #if [ -z "$PACKAGES_TO_INSTALL" ]; then - # spack fetch --missing # 2>&1 | tee log.fetch - #else - # spack fetch --missing --dependencies $PACKAGES_TO_INSTALL # 2>&1 | tee log.fetch - #fi - # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST # 2>&1 | tee log.test.install - # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL # 2>&1 | tee log.install -======= - if [ "${SOURCE_CACHE::7}" == "file://" ]; then - mirrorpath=${SOURCE_CACHE} - else - mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") - fi - spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch - # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST - # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL ->>>>>>> 5068f19444eaa1564c66a6e9ff751bf7fac16be6 + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/build-$envname + echo "Building environment $envname in $envdir" + cd $envdir + spack env activate . + if [ "${SOURCE_CACHE::7}" == "file://" ]; then + mirrorpath=${SOURCE_CACHE} + else + mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") + fi + spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch + # Just install the packages we're testing (+dependencies): + if [[ -z "${PACKAGES_TO_TEST}" ]]; then + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + fi + # Install the rest of the stack as usual: + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL + done done From d73425069d3616e36270ce434a369333f30863a3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 10:59:14 -0600 Subject: [PATCH 100/134] Fix bug in env name definition in uti/weekly_build/{03,04}* --- util/weekly_build/03_SetupEnv.sh | 2 +- util/weekly_build/04_SpackInstall.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index d9fe5a13c..9365a9b81 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -13,7 +13,7 @@ set -x for compiler in $COMPILERS; do for template in $TEMPLATES; do envname=build-$template-${compiler/@/-} - envdir=$RUNDIR/$RUNID/envs/build-$envname + envdir=$RUNDIR/$RUNID/envs/$envname echo "Setting up environment $envname in $envdir" rm -rf $envdir spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index 857d7f472..d83387453 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -21,7 +21,7 @@ INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" for compiler in $COMPILERS; do for template in $TEMPLATES; do envname=build-$template-${compiler/@/-} - envdir=$RUNDIR/$RUNID/envs/build-$envname + envdir=$RUNDIR/$RUNID/envs/$envname echo "Building environment $envname in $envdir" cd $envdir spack env activate . From a474cb74052717ef562c8d40e482abb8eed1da5d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 11:29:36 -0600 Subject: [PATCH 101/134] Strip down CI testing to neptune-dev w/o Python for faster development and debugging --- configs/templates/neptune-dev/spack.yaml | 2 +- util/weekly_build/sites/nautilus.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 9a9e3dfec..75dfeddb4 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -9,7 +9,7 @@ spack: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - neptune-env +espc ^esmf@8.7.0 - - neptune-python-env +xnrl ^esmf@8.7.0 + #- neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh index 916b799a7..9cd0dabd7 100644 --- a/util/weekly_build/sites/nautilus.sh +++ b/util/weekly_build/sites/nautilus.sh @@ -1,5 +1,6 @@ COMPILERS=${COMPILERS:-"intel oneapi gcc"} -TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"} +#TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"} +TEMPLATES=${TEMPLATES:-"neptune-dev"} module purge umask 0022 SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack From a803a546784c27684c7753dbcbf47edc927641e2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 16:04:18 -0600 Subject: [PATCH 102/134] Add steps 05_BuildCache and 07_Cleanup to Github actions, run only for OneAPI for testing --- .github/actions/Nightly_05_BuildCache/action.yaml | 11 +++++++++++ .github/actions/Nightly_07_Cleanup/action.yaml | 11 +++++++++++ ...ly-intel.yaml => nautilus-nightly-intel.yaml.tmp} | 2 +- .github/workflows/nautilus-nightly-oneapi.yaml | 7 ++++++- util/weekly_build/05_BuildCache.sh | 12 ++++++++---- util/weekly_build/sites/nautilus.sh | 2 ++ 6 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 .github/actions/Nightly_05_BuildCache/action.yaml create mode 100644 .github/actions/Nightly_07_Cleanup/action.yaml rename .github/workflows/{nautilus-nightly-intel.yaml => nautilus-nightly-intel.yaml.tmp} (94%) diff --git a/.github/actions/Nightly_05_BuildCache/action.yaml b/.github/actions/Nightly_05_BuildCache/action.yaml new file mode 100644 index 000000000..af064d659 --- /dev/null +++ b/.github/actions/Nightly_05_BuildCache/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 05 Build Cache + +runs: + using: "composite" + steps: + - name: Nightly 05 Build Cache + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_07_Cleanup/action.yaml b/.github/actions/Nightly_07_Cleanup/action.yaml new file mode 100644 index 000000000..70f27c4f1 --- /dev/null +++ b/.github/actions/Nightly_07_Cleanup/action.yaml @@ -0,0 +1,11 @@ +name: Nightly 07 Cleanup + +runs: + using: "composite" + steps: + - name: Nightly 07 Cleanup + shell: bash + run: | + RUNID=`date +"%Y%m%d"` + cd util/weekly_build + ./05_BuildCache.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml.tmp similarity index 94% rename from .github/workflows/nautilus-nightly-intel.yaml rename to .github/workflows/nautilus-nightly-intel.yaml.tmp index 90767bb18..b6ffd15fa 100644 --- a/.github/workflows/nautilus-nightly-intel.yaml +++ b/.github/workflows/nautilus-nightly-intel.yaml.tmp @@ -16,7 +16,7 @@ defaults: shell: bash env: - BASEDIR: /p/work2/heinzell/spack-stack-nightly + BASEDIR: /p/work2/heinzell/spack-stack-nightly/intel PLATFORM: nautilus COMPILERS: intel diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index cd8823cb0..c5d689444 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -16,7 +16,7 @@ defaults: shell: bash env: - BASEDIR: /p/work2/heinzell/spack-stack-nightly + BASEDIR: /p/work2/heinzell/spack-stack-nightly/oneapi PLATFORM: nautilus COMPILERS: oneapi @@ -39,3 +39,8 @@ jobs: uses: ./.github/actions/Nightly_03_SetupEnv - name: 04_SpackInstall uses: ./.github/actions/Nightly_04_SpackInstall + - name: 05_BuildCache + uses: ./.github/actions/Nightly_05_BuildCache + # Skip step 6 - app test - for the moment + - name: 07_Cleanup + uses: ./.github/actions/Nightly_07_Cleanup diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh index 4ce53cba7..2799c0464 100755 --- a/util/weekly_build/05_BuildCache.sh +++ b/util/weekly_build/05_BuildCache.sh @@ -11,8 +11,12 @@ set +x set -x for compiler in $COMPILERS; do - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL - spack buildcache rebuild-index ${BUILD_CACHE} + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + cd $envdir + spack env activate . + spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL + spack buildcache rebuild-index ${BUILD_CACHE} + done done diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh index 9cd0dabd7..5f76087d1 100644 --- a/util/weekly_build/sites/nautilus.sh +++ b/util/weekly_build/sites/nautilus.sh @@ -3,5 +3,7 @@ COMPILERS=${COMPILERS:-"intel oneapi gcc"} TEMPLATES=${TEMPLATES:-"neptune-dev"} module purge umask 0022 + SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack SPACK_STACK_BRANCH=feature/weekly_build_nautilus +KEEP_WEEKLY_BUILD_DIR="YES" \ No newline at end of file From 4b127283dfc12411259d69eb25e76499e4fba9ad Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 16:04:57 -0600 Subject: [PATCH 103/134] Bug fix for wrong source cache for Narwhal and Nautilus, addition of binary cache for Atlantis, Narwhal, Nautilus --- configs/sites/tier1/atlantis/mirrors.yaml | 19 ++----------------- configs/sites/tier1/narwhal/mirrors.yaml | 19 ++----------------- configs/sites/tier1/nautilus/mirrors.yaml | 19 ++----------------- 3 files changed, 6 insertions(+), 51 deletions(-) diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml index 4d363e009..d2371749e 100644 --- a/configs/sites/tier1/atlantis/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///neptune_diagnostics/spack-stack/source-cache/ - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///neptune_diagnostics/spack-stack/source-cache/ - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///neptune_diagnostics/spack-stack/source-cache + local-binary: file:///neptune_diagnostics/spack-stack/build-cache diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index ec3db4a00..5b06fc3fb 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index ec3db4a00..5b06fc3fb 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache From 7b17952994f8d4b0c2a2328249ea0e153edf4c24 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 16:51:21 -0600 Subject: [PATCH 104/134] In util/weekly_build/{03_SetupEnv.sh,04_SpackInstall.sh} pipe spack concretizer and installer output to logfile, but don't redirect stderr --- util/weekly_build/03_SetupEnv.sh | 9 ++------- util/weekly_build/04_SpackInstall.sh | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 9365a9b81..5ad8a69c6 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -21,15 +21,10 @@ for compiler in $COMPILERS; do spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: - spack concretize --fresh 2>&1 | tee log.concretize + spack concretize --fresh | tee log.concretize ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. -# spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test -### # Get path to local-source mirror from spack site config -### SOURCE_MIRROR=`spack mirror list | grep local-source` -### SOURCE_MIRROR=${SOURCE_MIRROR/local-source \[sb\] file:\/\//} -### # Download all source codes needed to build the environment -### spack mirror create -a -d ${SOURCE_MIRROR} +# spack concretize --force --fresh --test all | tee log.concretize_test done done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index d83387453..c2d698d95 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -33,9 +33,9 @@ for compiler in $COMPILERS; do spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch # Just install the packages we're testing (+dependencies): if [[ -z "${PACKAGES_TO_TEST}" ]]; then - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test fi # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install done done From 9ca24073e05e33e99a116abf790b69af6f001b15 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 16:51:54 -0600 Subject: [PATCH 105/134] Add logic to .github/actions/Nightly_04_SpackInstall/action.yaml to reuse existing binary cache for spack-stack installs on all days except Sundays --- .github/actions/Nightly_04_SpackInstall/action.yaml | 9 +++++++++ .github/workflows/nautilus-nightly-oneapi.yaml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml index 0954971b1..bd2ab560d 100644 --- a/.github/actions/Nightly_04_SpackInstall/action.yaml +++ b/.github/actions/Nightly_04_SpackInstall/action.yaml @@ -6,6 +6,15 @@ runs: - name: Nightly 04 Spack Install shell: bash run: | + # Get day of week - build everything from scratch on Sundays, + # otherwise use build cache to speed up the installations + DOW=$(date +%u) + # Monday is 1 ... Sunday is 7 + if [[ $DOW == 7 ]]; then + export REUSE_BUILD_CACHE="NO" + else + export REUSE_BUILD_CACHE="YES" + fi RUNID=`date +"%Y%m%d"` cd util/weekly_build ./04_SpackInstall.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml index c5d689444..4459ceb4d 100644 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ b/.github/workflows/nautilus-nightly-oneapi.yaml @@ -6,6 +6,9 @@ on: branches: - develop types: [opened, synchronize, reopened] + # Is this UTC? + #schedule: + # - cron: '0 8 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} From 4c906cf767992b42431802913acb270f35b55825 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 05:49:35 -0600 Subject: [PATCH 106/134] Write RUNID and DOW to file in step Nightly_01_DirectorySetup, retrieve from files in other steps --- .../actions/Nightly_01_DirectorySetup/action.yaml | 13 ++++++++----- .../actions/Nightly_02_GetSpackStack/action.yaml | 4 +--- .github/actions/Nightly_03_SetupEnv/action.yaml | 2 +- .github/actions/Nightly_04_SpackInstall/action.yaml | 8 ++++---- .github/actions/Nightly_05_BuildCache/action.yaml | 2 +- .github/actions/Nightly_07_Cleanup/action.yaml | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index 4d1a614e9..2660a3745 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -6,13 +6,16 @@ runs: - name: Nightly 01 Directory Setup shell: bash run: | - RUNID=`date +"%Y%m%d"` # Write RUNID to file so that following steps use the # same date and don't pick up the next day's date + RUNID=`date +"%Y%m%d"` echo "${RUNID}" > RUNID_SAVE.log - # DH* TODO REMOVE LATER? - RUNDIR=${BASEDIR}/${RUNID} - rm -vfr ${RUNDIR} - # *DH + # Get day of week for later use and write to file + DOW=$(date +%u) + echo "${DOW}" > DOW_SAVE.log + ## DH* TODO REMOVE LATER? + #RUNDIR=${BASEDIR}/${RUNID} + #rm -vfr ${RUNDIR} + ## *DH cd util/weekly_build ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml index 49fbf2512..7377bfb88 100644 --- a/.github/actions/Nightly_02_GetSpackStack/action.yaml +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -6,9 +6,7 @@ runs: - name: Nightly 02 Get Spack-Stack shell: bash run: | - #RUNID=`date +"%Y%m%d"` RUNID=$( Date: Tue, 22 Oct 2024 07:09:47 -0600 Subject: [PATCH 107/134] Bug fix in util/weekly_build/04_SpackInstall.sh, add missing NOT (!) in if test for PACKAGES_TO_TEST --- util/weekly_build/04_SpackInstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index c2d698d95..eaac57dfd 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -32,7 +32,7 @@ for compiler in $COMPILERS; do fi spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch # Just install the packages we're testing (+dependencies): - if [[ -z "${PACKAGES_TO_TEST}" ]]; then + if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test fi # Install the rest of the stack as usual: From c32cb03c370d3727f11571d289cbf67ec5f5ba6f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 09:30:34 -0600 Subject: [PATCH 108/134] Update util/weekly_build/sites/{acorn.sh,nautilus.sh} and delete currently unused sites --- util/weekly_build/sites/acorn.sh | 1 + util/weekly_build/sites/derecho.sh | 1 - util/weekly_build/sites/discover16.sh | 1 - util/weekly_build/sites/discover17.sh | 1 - util/weekly_build/sites/gaea.sh | 1 - util/weekly_build/sites/hera.sh | 1 - util/weekly_build/sites/hercules.sh | 1 - util/weekly_build/sites/jet.sh | 1 - util/weekly_build/sites/linux.default.sh | 1 - util/weekly_build/sites/narwhal.sh | 1 - util/weekly_build/sites/nautilus.sh | 9 ++++++++- util/weekly_build/sites/orion.sh | 1 - util/weekly_build/sites/s4.sh | 1 - 13 files changed, 9 insertions(+), 12 deletions(-) delete mode 100644 util/weekly_build/sites/derecho.sh delete mode 100644 util/weekly_build/sites/discover16.sh delete mode 100644 util/weekly_build/sites/discover17.sh delete mode 100644 util/weekly_build/sites/gaea.sh delete mode 100644 util/weekly_build/sites/hera.sh delete mode 100644 util/weekly_build/sites/hercules.sh delete mode 100644 util/weekly_build/sites/jet.sh delete mode 100644 util/weekly_build/sites/linux.default.sh delete mode 100644 util/weekly_build/sites/narwhal.sh delete mode 100644 util/weekly_build/sites/orion.sh delete mode 100644 util/weekly_build/sites/s4.sh diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index ef19a4962..d84094f7b 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -1,5 +1,6 @@ module load gcc/11.2.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} +TEMPLATES=${TEMPLATES:-"unified-dev"} function spack_install_exe { # set +e # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/derecho.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/discover16.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/discover17.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh deleted file mode 100644 index fe7e69664..000000000 --- a/util/weekly_build/sites/gaea.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel"} diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/hera.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/hercules.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/jet.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/linux.default.sh b/util/weekly_build/sites/linux.default.sh deleted file mode 100644 index c044afced..000000000 --- a/util/weekly_build/sites/linux.default.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"gcc"} diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/narwhal.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/nautilus.sh b/util/weekly_build/sites/nautilus.sh index fe7e69664..fddcfd62f 100644 --- a/util/weekly_build/sites/nautilus.sh +++ b/util/weekly_build/sites/nautilus.sh @@ -1 +1,8 @@ -COMPILERS=${COMPILERS:-"intel"} +COMPILERS=${COMPILERS:-"intel oneapi gcc"} +TEMPLATES=${TEMPLATES:-"neptune-dev unified-dev"} +module purge +umask 0022 + +SPACK_STACK_URL=https://github.nrlmry.navy.mil/JCSDA/spack-stack +SPACK_STACK_BRANCH=feature/weekly_build_nautilus +KEEP_WEEKLY_BUILD_DIR="YES" diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/orion.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh deleted file mode 100644 index fe7e69664..000000000 --- a/util/weekly_build/sites/s4.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel"} From 31d378d71a15fc9f87b26df33c5498d056090ccc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 09:30:51 -0600 Subject: [PATCH 109/134] Update mirrors for NRL machines in configs/sites/tier1 --- configs/sites/tier1/atlantis/mirrors.yaml | 19 ++----------------- configs/sites/tier1/narwhal/mirrors.yaml | 19 ++----------------- configs/sites/tier1/nautilus/mirrors.yaml | 19 ++----------------- 3 files changed, 6 insertions(+), 51 deletions(-) diff --git a/configs/sites/tier1/atlantis/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml index 4d363e009..d2371749e 100644 --- a/configs/sites/tier1/atlantis/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///neptune_diagnostics/spack-stack/source-cache/ - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///neptune_diagnostics/spack-stack/source-cache/ - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///neptune_diagnostics/spack-stack/source-cache + local-binary: file:///neptune_diagnostics/spack-stack/build-cache diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 2b67bea03..5b06fc3fb 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 2b67bea03..5b06fc3fb 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,18 +1,3 @@ mirrors: - local-source: - fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null - push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache - access_pair: - - null - - null - access_token: null - profile: null - endpoint_url: null + local-source: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + local-binary: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache From 43ab602ab05d06c91d143f3666bf76f741db8364 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 09:31:42 -0600 Subject: [PATCH 110/134] Update util/weekly_build scripts to handle multiple environments plus other updates --- util/weekly_build/03_SetupEnv.sh | 30 ++++++++++++++++++ util/weekly_build/03_SetupUnifiedEnv.sh | 24 -------------- util/weekly_build/04_SpackInstall.sh | 31 ++++++++++++------- util/weekly_build/05_BuildCache.sh | 12 ++++--- util/weekly_build/ShellSetup.sh | 6 ++-- .../SpackStackBuildCache_AllSteps.sh | 2 +- 6 files changed, 62 insertions(+), 43 deletions(-) create mode 100755 util/weekly_build/03_SetupEnv.sh delete mode 100755 util/weekly_build/03_SetupUnifiedEnv.sh diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh new file mode 100755 index 000000000..5ad8a69c6 --- /dev/null +++ b/util/weekly_build/03_SetupEnv.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + +set -ex + +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + echo "Setting up environment $envname in $envdir" + rm -rf $envdir + spack stack create env --name build-${template}-${compiler/@/-} --template $template --site $PLATFORM --compiler $compiler + cd $envdir + spack env activate . + spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" + # Check for duplicates and fail before doing the "real" concretization: + spack concretize --fresh | tee log.concretize + ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf + spack concretize --fresh --force +# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. +# spack concretize --force --fresh --test all | tee log.concretize_test + done +done diff --git a/util/weekly_build/03_SetupUnifiedEnv.sh b/util/weekly_build/03_SetupUnifiedEnv.sh deleted file mode 100755 index 3479ed7ec..000000000 --- a/util/weekly_build/03_SetupUnifiedEnv.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi - -set -ex - -cd $RUNDIR/$RUNID - -set +x -. setup.sh -set -x - -for compiler in $COMPILERS; do - rm -rf $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack stack create env --name build-${compiler/@/-} --template unified-dev --site $PLATFORM --compiler $compiler - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" - # Check for duplicates and fail before doing the "real" concretization with test deps: - spack concretize --fresh 2>&1 | tee log.concretize - ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf -# The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. -# spack concretize --force --fresh --test all 2>&1 | tee log.concretize_test -done diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index a5bb8cc70..eaac57dfd 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -19,16 +19,23 @@ fi INSTALL_OPTS="--show-log-on-error --fail-fast $cache_flag $INSTALL_OPTS" for compiler in $COMPILERS; do - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - if [ "${SOURCE_CACHE::7}" == "file://" ]; then - mirrorpath=${SOURCE_CACHE} - else - mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") - fi - spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch - # Just install the packages we're testing (+dependencies): - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST - # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + echo "Building environment $envname in $envdir" + cd $envdir + spack env activate . + if [ "${SOURCE_CACHE::7}" == "file://" ]; then + mirrorpath=${SOURCE_CACHE} + else + mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") + fi + spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch + # Just install the packages we're testing (+dependencies): + if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then + spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test + fi + # Install the rest of the stack as usual: + spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install + done done diff --git a/util/weekly_build/05_BuildCache.sh b/util/weekly_build/05_BuildCache.sh index 4ce53cba7..2799c0464 100755 --- a/util/weekly_build/05_BuildCache.sh +++ b/util/weekly_build/05_BuildCache.sh @@ -11,8 +11,12 @@ set +x set -x for compiler in $COMPILERS; do - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL - spack buildcache rebuild-index ${BUILD_CACHE} + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + cd $envdir + spack env activate . + spack buildcache push --unsigned --force ${BUILD_CACHE} $PACKAGES_TO_INSTALL + spack buildcache rebuild-index ${BUILD_CACHE} + done done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 31e7f2e8e..e5f25608a 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -11,16 +11,18 @@ if [ ${RUNDIR::1} != "/" ]; then exit 1 fi -PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} +#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} function alert_cmd { echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." } function spack_install_exe { - spack $* | tee -a log.install 2>&1 + #spack $* | tee -a log.install 2>&1 + spack $* } +# Include platform-dependent configuration . $(dirname $0)/sites/${PLATFORM}.sh SOURCE_CACHE=${SOURCE_CACHE:-local-source} diff --git a/util/weekly_build/SpackStackBuildCache_AllSteps.sh b/util/weekly_build/SpackStackBuildCache_AllSteps.sh index 4d26df07d..73a7931c4 100755 --- a/util/weekly_build/SpackStackBuildCache_AllSteps.sh +++ b/util/weekly_build/SpackStackBuildCache_AllSteps.sh @@ -18,7 +18,7 @@ function trap_and_run { trap_and_run ./01_DirectorySetup.sh $* trap_and_run ./02_GetSpackStack.sh $* -trap_and_run ./03_SetupUnifiedEnv.sh $* +trap_and_run ./03_SetupEnv.sh $* trap_and_run ./04_SpackInstall.sh $* trap_and_run ./05_BuildCache.sh $* trap_and_run ./06_AppTests.sh $* From f09b991b3a4148b08cd916cde1d03aef74578a39 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 09:32:13 -0600 Subject: [PATCH 111/134] Add .github/actions/Nightly_* and .github/workflows/nautilus-nightly-oneapi.yaml --- .../Nightly_01_DirectorySetup/action.yaml | 21 ++++++++ .../Nightly_02_GetSpackStack/action.yaml | 12 +++++ .../actions/Nightly_03_SetupEnv/action.yaml | 11 +++++ .../Nightly_04_SpackInstall/action.yaml | 20 ++++++++ .../actions/Nightly_05_BuildCache/action.yaml | 11 +++++ .../actions/Nightly_07_Cleanup/action.yaml | 11 +++++ .../workflows/nautilus-nightly-oneapi.yaml | 49 +++++++++++++++++++ 7 files changed, 135 insertions(+) create mode 100644 .github/actions/Nightly_01_DirectorySetup/action.yaml create mode 100644 .github/actions/Nightly_02_GetSpackStack/action.yaml create mode 100644 .github/actions/Nightly_03_SetupEnv/action.yaml create mode 100644 .github/actions/Nightly_04_SpackInstall/action.yaml create mode 100644 .github/actions/Nightly_05_BuildCache/action.yaml create mode 100644 .github/actions/Nightly_07_Cleanup/action.yaml create mode 100644 .github/workflows/nautilus-nightly-oneapi.yaml diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml new file mode 100644 index 000000000..2660a3745 --- /dev/null +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -0,0 +1,21 @@ +name: Nightly 01 Directory Setup + +runs: + using: "composite" + steps: + - name: Nightly 01 Directory Setup + shell: bash + run: | + # Write RUNID to file so that following steps use the + # same date and don't pick up the next day's date + RUNID=`date +"%Y%m%d"` + echo "${RUNID}" > RUNID_SAVE.log + # Get day of week for later use and write to file + DOW=$(date +%u) + echo "${DOW}" > DOW_SAVE.log + ## DH* TODO REMOVE LATER? + #RUNDIR=${BASEDIR}/${RUNID} + #rm -vfr ${RUNDIR} + ## *DH + cd util/weekly_build + ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml new file mode 100644 index 000000000..7377bfb88 --- /dev/null +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -0,0 +1,12 @@ +name: Nightly 02 Get Spack-Stack + +runs: + using: "composite" + steps: + - name: Nightly 02 Get Spack-Stack + shell: bash + run: | + RUNID=$( Date: Tue, 22 Oct 2024 14:55:59 -0600 Subject: [PATCH 112/134] Clean up .github/actions/** --- .github/actions/Nightly_01_DirectorySetup/action.yaml | 5 +---- .github/actions/Nightly_02_GetSpackStack/action.yaml | 1 - 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index 2660a3745..5bcf5051c 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -13,9 +13,6 @@ runs: # Get day of week for later use and write to file DOW=$(date +%u) echo "${DOW}" > DOW_SAVE.log - ## DH* TODO REMOVE LATER? - #RUNDIR=${BASEDIR}/${RUNID} - #rm -vfr ${RUNDIR} - ## *DH + # cd util/weekly_build ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml index 7377bfb88..268adaf29 100644 --- a/.github/actions/Nightly_02_GetSpackStack/action.yaml +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -7,6 +7,5 @@ runs: shell: bash run: | RUNID=$( Date: Tue, 22 Oct 2024 14:57:28 -0600 Subject: [PATCH 113/134] Remove sites that are not in use from util/weekly_build/sites/ --- configs/common/packages | 0 util/weekly_build/sites/derecho.sh | 1 - util/weekly_build/sites/discover16.sh | 1 - util/weekly_build/sites/discover17.sh | 1 - util/weekly_build/sites/gaea.sh | 1 - util/weekly_build/sites/hera.sh | 1 - util/weekly_build/sites/hercules.sh | 1 - util/weekly_build/sites/jet.sh | 1 - util/weekly_build/sites/linux.default.sh | 1 - util/weekly_build/sites/narwhal.sh | 1 - util/weekly_build/sites/orion.sh | 1 - util/weekly_build/sites/s4.sh | 1 - 12 files changed, 11 deletions(-) delete mode 100644 configs/common/packages delete mode 100644 util/weekly_build/sites/derecho.sh delete mode 100644 util/weekly_build/sites/discover16.sh delete mode 100644 util/weekly_build/sites/discover17.sh delete mode 100644 util/weekly_build/sites/gaea.sh delete mode 100644 util/weekly_build/sites/hera.sh delete mode 100644 util/weekly_build/sites/hercules.sh delete mode 100644 util/weekly_build/sites/jet.sh delete mode 100644 util/weekly_build/sites/linux.default.sh delete mode 100644 util/weekly_build/sites/narwhal.sh delete mode 100644 util/weekly_build/sites/orion.sh delete mode 100644 util/weekly_build/sites/s4.sh diff --git a/configs/common/packages b/configs/common/packages deleted file mode 100644 index e69de29bb..000000000 diff --git a/util/weekly_build/sites/derecho.sh b/util/weekly_build/sites/derecho.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/derecho.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/discover16.sh b/util/weekly_build/sites/discover16.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/discover16.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/discover17.sh b/util/weekly_build/sites/discover17.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/discover17.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/gaea.sh b/util/weekly_build/sites/gaea.sh deleted file mode 100644 index fe7e69664..000000000 --- a/util/weekly_build/sites/gaea.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel"} diff --git a/util/weekly_build/sites/hera.sh b/util/weekly_build/sites/hera.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/hera.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/hercules.sh b/util/weekly_build/sites/hercules.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/hercules.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/jet.sh b/util/weekly_build/sites/jet.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/jet.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/linux.default.sh b/util/weekly_build/sites/linux.default.sh deleted file mode 100644 index c044afced..000000000 --- a/util/weekly_build/sites/linux.default.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"gcc"} diff --git a/util/weekly_build/sites/narwhal.sh b/util/weekly_build/sites/narwhal.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/narwhal.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh deleted file mode 100644 index 1e7dc8450..000000000 --- a/util/weekly_build/sites/orion.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel gcc"} diff --git a/util/weekly_build/sites/s4.sh b/util/weekly_build/sites/s4.sh deleted file mode 100644 index fe7e69664..000000000 --- a/util/weekly_build/sites/s4.sh +++ /dev/null @@ -1 +0,0 @@ -COMPILERS=${COMPILERS:-"intel"} From edab0f563178a0c4342e91bc3b3d42013db647e8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 14:57:53 -0600 Subject: [PATCH 114/134] Add rogue cmd line argument for spack concretize for debugging in util/weekly_build/03_SetupEnv.sh --- util/weekly_build/03_SetupEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 5ad8a69c6..9a1991046 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -21,7 +21,7 @@ for compiler in $COMPILERS; do spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: - spack concretize --fresh | tee log.concretize + spack concretize --fresh --somethingnotsupported > log.concretize 2>&1 ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. From 8c895401c7946c3bf5168df83d11a609aa19f56a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 15:58:27 -0600 Subject: [PATCH 115/134] Try spack_wrapper --- util/weekly_build/03_SetupEnv.sh | 6 +++++- util/weekly_build/04_SpackInstall.sh | 8 +++++--- util/weekly_build/ShellSetup.sh | 7 +++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 9a1991046..b8810cb7c 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -21,8 +21,12 @@ for compiler in $COMPILERS; do spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: - spack concretize --fresh --somethingnotsupported > log.concretize 2>&1 + #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1 + spack_wrapper log.concretize concretize --fresh --somethingnotsupported ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf + + echo "EXIT HERE FOR TESTING" + exit 1 spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all | tee log.concretize_test diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index eaac57dfd..faeca8456 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -30,12 +30,14 @@ for compiler in $COMPILERS; do else mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") fi - spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch + spack_wrapper log.fetch mirror create --dependencies \ + --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \ + ${PACKAGES_TO_INSTALL:---all} # Just install the packages we're testing (+dependencies): if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test + spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST fi # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install + spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL done done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index e5f25608a..106965dee 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -22,6 +22,13 @@ function spack_install_exe { spack $* } +function spack_wrapper { + logfile=$1 + shift + set -o pipefail + spack $* 2>&1 | tee -a $logfile +} + # Include platform-dependent configuration . $(dirname $0)/sites/${PLATFORM}.sh From c9d0fb2827455ac5865c7db05474791143f2f21c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 20:42:34 -0600 Subject: [PATCH 116/134] Remove debugging code --- util/weekly_build/03_SetupEnv.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index b8810cb7c..6dc944d09 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -22,11 +22,8 @@ for compiler in $COMPILERS; do spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1 - spack_wrapper log.concretize concretize --fresh --somethingnotsupported + spack_wrapper log.concretize concretize --fresh ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf - - echo "EXIT HERE FOR TESTING" - exit 1 spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all | tee log.concretize_test From 5105cf03de445bf1e9185450d1a1fb6f42bd7ebe Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Oct 2024 05:36:24 -0600 Subject: [PATCH 117/134] Clean up util/weekly_build/03_SetupEnv.sh, util/weekly_build/ShellSetup.sh, and re-enable .github/workflows/nautilus-nightly-intel.yaml --- ...nightly-intel.yaml.tmp => nautilus-nightly-intel.yaml} | 8 ++++++++ util/weekly_build/03_SetupEnv.sh | 1 - util/weekly_build/ShellSetup.sh | 8 ++------ 3 files changed, 10 insertions(+), 7 deletions(-) rename .github/workflows/{nautilus-nightly-intel.yaml.tmp => nautilus-nightly-intel.yaml} (79%) diff --git a/.github/workflows/nautilus-nightly-intel.yaml.tmp b/.github/workflows/nautilus-nightly-intel.yaml similarity index 79% rename from .github/workflows/nautilus-nightly-intel.yaml.tmp rename to .github/workflows/nautilus-nightly-intel.yaml index b6ffd15fa..5225263bf 100644 --- a/.github/workflows/nautilus-nightly-intel.yaml.tmp +++ b/.github/workflows/nautilus-nightly-intel.yaml @@ -6,6 +6,9 @@ on: branches: - develop types: [opened, synchronize, reopened] + # Is this UTC? + #schedule: + # - cron: '0 8 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -39,3 +42,8 @@ jobs: uses: ./.github/actions/Nightly_03_SetupEnv - name: 04_SpackInstall uses: ./.github/actions/Nightly_04_SpackInstall + - name: 05_BuildCache + uses: ./.github/actions/Nightly_05_BuildCache + # Skip step 6 - app test - for the moment + - name: 07_Cleanup + uses: ./.github/actions/Nightly_07_Cleanup diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 6dc944d09..8bfa74b13 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -21,7 +21,6 @@ for compiler in $COMPILERS; do spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: - #spack concretize --fresh --somethingnotsupported > log.concretize 2>&1 spack_wrapper log.concretize concretize --fresh ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf spack concretize --fresh --force diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 106965dee..5091f214c 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -11,17 +11,13 @@ if [ ${RUNDIR::1} != "/" ]; then exit 1 fi -#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} +# Defaults to empty, can be overwritten in site config +PACKAGES_TO_TEST="" function alert_cmd { echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." } -function spack_install_exe { - #spack $* | tee -a log.install 2>&1 - spack $* -} - function spack_wrapper { logfile=$1 shift From efb23cc5546be7579917dc68e39b9de58040b5b0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Oct 2024 05:37:17 -0600 Subject: [PATCH 118/134] Introduce spack_wrapper calls --- util/weekly_build/03_SetupEnv.sh | 2 +- util/weekly_build/04_SpackInstall.sh | 8 +++++--- util/weekly_build/ShellSetup.sh | 11 +++++++---- util/weekly_build/sites/acorn.sh | 1 + 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 5ad8a69c6..8bfa74b13 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -21,7 +21,7 @@ for compiler in $COMPILERS; do spack env activate . spack config add "config:install_tree:padded_length:${PADDED_LENGTH:-200}" # Check for duplicates and fail before doing the "real" concretization: - spack concretize --fresh | tee log.concretize + spack_wrapper log.concretize concretize --fresh ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index eaac57dfd..faeca8456 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -30,12 +30,14 @@ for compiler in $COMPILERS; do else mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") fi - spack mirror create --dependencies --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} ${PACKAGES_TO_INSTALL:---all} 2>&1 | tee log.fetch + spack_wrapper log.fetch mirror create --dependencies \ + --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \ + ${PACKAGES_TO_INSTALL:---all} # Just install the packages we're testing (+dependencies): if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then - spack_install_exe install $INSTALL_OPTS --test root $PACKAGES_TO_TEST | tee log.install-and-test + spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST fi # Install the rest of the stack as usual: - spack_install_exe install $INSTALL_OPTS $PACKAGES_TO_INSTALL | tee log.install + spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL done done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index e5f25608a..5091f214c 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -11,15 +11,18 @@ if [ ${RUNDIR::1} != "/" ]; then exit 1 fi -#PACKAGES_TO_TEST=${PACKAGES_TO_TEST:-"libpng libaec jasper scotch w3emc g2 g2c"} +# Defaults to empty, can be overwritten in site config +PACKAGES_TO_TEST="" function alert_cmd { echo "Your run failed in $1. This is a placeholder alerting function. 'alert_cmd' should be defined for each system." } -function spack_install_exe { - #spack $* | tee -a log.install 2>&1 - spack $* +function spack_wrapper { + logfile=$1 + shift + set -o pipefail + spack $* 2>&1 | tee -a $logfile } # Include platform-dependent configuration diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index d84094f7b..d795400c4 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -1,6 +1,7 @@ module load gcc/11.2.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} TEMPLATES=${TEMPLATES:-"unified-dev"} +# TODO REPLACE WITH spack_wrapper function spack_install_exe { # set +e # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & From d75cb5b511cfe7ed95ab82c77352c20ae12579af Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 23 Oct 2024 05:38:02 -0600 Subject: [PATCH 119/134] Update github actions and workflows --- .../Nightly_01_DirectorySetup/action.yaml | 5 +- .../Nightly_02_GetSpackStack/action.yaml | 1 - .github/workflows/nautilus-nightly-intel.yaml | 49 +++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/nautilus-nightly-intel.yaml diff --git a/.github/actions/Nightly_01_DirectorySetup/action.yaml b/.github/actions/Nightly_01_DirectorySetup/action.yaml index 2660a3745..5bcf5051c 100644 --- a/.github/actions/Nightly_01_DirectorySetup/action.yaml +++ b/.github/actions/Nightly_01_DirectorySetup/action.yaml @@ -13,9 +13,6 @@ runs: # Get day of week for later use and write to file DOW=$(date +%u) echo "${DOW}" > DOW_SAVE.log - ## DH* TODO REMOVE LATER? - #RUNDIR=${BASEDIR}/${RUNID} - #rm -vfr ${RUNDIR} - ## *DH + # cd util/weekly_build ./01_DirectorySetup.sh ${RUNID} ${BASEDIR} ${PLATFORM} diff --git a/.github/actions/Nightly_02_GetSpackStack/action.yaml b/.github/actions/Nightly_02_GetSpackStack/action.yaml index 7377bfb88..268adaf29 100644 --- a/.github/actions/Nightly_02_GetSpackStack/action.yaml +++ b/.github/actions/Nightly_02_GetSpackStack/action.yaml @@ -7,6 +7,5 @@ runs: shell: bash run: | RUNID=$( Date: Wed, 23 Oct 2024 05:40:58 -0600 Subject: [PATCH 120/134] Update .github/actions/Nightly_04_SpackInstall/action.yaml --- .github/actions/Nightly_04_SpackInstall/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/Nightly_04_SpackInstall/action.yaml b/.github/actions/Nightly_04_SpackInstall/action.yaml index 3d86123e1..b6472ccda 100644 --- a/.github/actions/Nightly_04_SpackInstall/action.yaml +++ b/.github/actions/Nightly_04_SpackInstall/action.yaml @@ -8,7 +8,7 @@ runs: run: | RUNID=$( Date: Tue, 5 Nov 2024 11:13:55 -0700 Subject: [PATCH 121/134] Remove extraneous spack concretize command from util/weekly_build/03_SetupEnv.sh --- util/weekly_build/03_SetupEnv.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/util/weekly_build/03_SetupEnv.sh b/util/weekly_build/03_SetupEnv.sh index 8bfa74b13..a98fa6cdd 100755 --- a/util/weekly_build/03_SetupEnv.sh +++ b/util/weekly_build/03_SetupEnv.sh @@ -23,7 +23,6 @@ for compiler in $COMPILERS; do # Check for duplicates and fail before doing the "real" concretization: spack_wrapper log.concretize concretize --fresh ${SPACK_STACK_DIR:?}/util/show_duplicate_packages.py log.concretize -d -i crtm -i esmf - spack concretize --fresh --force # The following is not working at the moment, for seemingly a couple reasons. Therefore packages with test-only deps cannot be tested. # spack concretize --force --fresh --test all | tee log.concretize_test done From e63117e8df4a99bb84bcfe55c68cb43b2fc0fa2e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Nov 2024 11:15:02 -0700 Subject: [PATCH 122/134] Reinstantiate spack_install_exe as spack_install_wrapper --- util/weekly_build/04_SpackInstall.sh | 4 ++-- util/weekly_build/ShellSetup.sh | 7 +++++++ util/weekly_build/sites/acorn.sh | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index faeca8456..a40a3f6f1 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -35,9 +35,9 @@ for compiler in $COMPILERS; do ${PACKAGES_TO_INSTALL:---all} # Just install the packages we're testing (+dependencies): if [[ ! -z "${PACKAGES_TO_TEST}" ]]; then - spack_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST + spack_install_wrapper log.install-and-test install $INSTALL_OPTS --test root $PACKAGES_TO_TEST fi # Install the rest of the stack as usual: - spack_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL + spack_install_wrapper log.install install $INSTALL_OPTS $PACKAGES_TO_INSTALL done done diff --git a/util/weekly_build/ShellSetup.sh b/util/weekly_build/ShellSetup.sh index 5091f214c..c516e6fb3 100644 --- a/util/weekly_build/ShellSetup.sh +++ b/util/weekly_build/ShellSetup.sh @@ -25,6 +25,13 @@ function spack_wrapper { spack $* 2>&1 | tee -a $logfile } +function spack_install_wrapper { + logfile=$1 + shift + set -o pipefail + spack $* 2>&1 | tee -a $logfile +} + # Include platform-dependent configuration . $(dirname $0)/sites/${PLATFORM}.sh diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index d84094f7b..f20f7da29 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -1,7 +1,9 @@ module load gcc/11.2.0 python/3.11.7 COMPILERS=${COMPILERS:-"intel@2022.0.2.262 intel@19.1.3.304"} TEMPLATES=${TEMPLATES:-"unified-dev"} -function spack_install_exe { +function spack_install_wrapper { + logfile=$1 + shift # set +e # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-A -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & # ( /opt/pbs/bin/qsub -N spack-build-cache-$RUNID-B -j oe -A NCEPLIBS-DEV -l select=1:ncpus=6:mem=10000MB -l walltime=03:00:00 -V -Wblock=true -- $(which spack) $* ) & From 3dfba8165bbd8bd0bccba4bb36cb20842b81f53c Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 13 Nov 2024 13:03:58 -0800 Subject: [PATCH 123/134] Update 02_GetSpackStack.sh --- util/weekly_build/02_GetSpackStack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/weekly_build/02_GetSpackStack.sh b/util/weekly_build/02_GetSpackStack.sh index 5fe77f7cc..f1eab34eb 100755 --- a/util/weekly_build/02_GetSpackStack.sh +++ b/util/weekly_build/02_GetSpackStack.sh @@ -9,5 +9,5 @@ if [ -d $RUNID ]; then cd $RUNID git pull else - git clone --depth 1 --recurse-submodules --shallow-submodules ${SPACK_STACK_URL:-https://github.com/AlexanderRichert-NOAA/spack-stack} -b ${SPACK_STACK_BRANCH:-weekly_build} $RUNID + git clone --depth 1 --recurse-submodules --shallow-submodules ${SPACK_STACK_URL:-https://github.com/JCSDA/spack-stack} -b ${SPACK_STACK_BRANCH:-develop} $RUNID fi From 08e9647d191190592d1a7132e895979ba1b489a6 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 13 Nov 2024 14:10:35 -0800 Subject: [PATCH 124/134] add AutomatedBuilds.rst to docs --- doc/source/AutomatedBuilds.rst | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 doc/source/AutomatedBuilds.rst diff --git a/doc/source/AutomatedBuilds.rst b/doc/source/AutomatedBuilds.rst new file mode 100644 index 000000000..18eab61e4 --- /dev/null +++ b/doc/source/AutomatedBuilds.rst @@ -0,0 +1,54 @@ +================ +Automated builds +================ + +.. _AutomatedBuildWorkflow: + +Workflow +************************* + +The spack-stack repository now contains scripts for implementing scheduled, automated building, unit testing, and build caching of packages. These scripts live under utils/weekly_build/. They download their own copy of spack-stack (the develop branch, by default) and build one or more environments under that directory structure, which may be transient or reused. + +Script failures are detected by non-zero exit codes. + +The list of packages to be unit tested is stored in the ``$PACKAGES_TO_TEST`` variable in ShellSetup.sh. As unit testing needs are identified, this list should be updated to be as comprehensive as possible based on https://github.com/JCSDA/spack-stack/wiki/Automated-testing-and-building#unit-testing-requirements. + +.. _AutomatedBuildAddPlatform: + +Overview of implementing on a new platform +****************************************** + +Use a clean environment. Do NOT run setup.sh in the directory containing the build scripts (util/weekly_build/) to be run. Nothing in spack-stack outside of the build scripts is needed to get the automated build workflow running, as the workflow will take care of download a copy of spack-stack to be used for doing the automated builds themselves. + +------------------------------- +Platform-specific configuration +------------------------------- + +The ShellSetup.sh script points to a script in the sites/ subdirectory based on the site name as set by the third argument of each build script (stored and used as ``$PLATFORM`` throughout the scripts). Note that a number of variables and functions can be set or overridden here: + - ``$COMPILERS``: Space-delimited list of compilers to be built against on a given system. + - ``$TEMPLATES``: Space-delimited list of templates to install. Default is 'unified-env'. + - ``$PACKAGES_TO_TEST``: Space-delimited list of packages for which units tests should be run. + - ``$PACKAGES_TO_INSTALL``: Space-delimited list of packages to be installed. For many systems it can be left empty so as to install the entire Unified Environment. + - ``alert_cmd()``: This function should be set for each system. It is invoked when any step fails (non-zero exit code somewhere in the workflow), and so can be used to send emails, post to Slack or GitHub, etc. + - ``spack_wrapper()``: This function runs the Spack command, where the first argument is a log file name. It should not typically need to be redefined by platform. + - ``spack_install_wrapper()``: This function determines the means by which the ``spack`` command is invoked for the install steps, and can be overridden for each platform. If testing and installation will be performed outside of a batch scheduler, i.e., directly on a login node, then the default can probably be used. However, overriding this function allows, for instance, for job schedulers to be invoked (srun, qsub, etc.), as well as for parallel installations to be performed. Note that the first argument is a log file name, so custom versions of this function will most likely need to use ``shift`` to properly access the remaining arguments. + - ``$INSTALL_OPTS``: Additional ``spack install`` flags to be supplied to ``spack_install_wrapper()``. + - ``$TEST_``: Set this variable to ON in order to enable the corresponding test under util/weekly_build/apptests/. + - ``$BUILD_CACHE``: Name of or path to (beginning with '\file://') build cache directory. Defaults to 'local-binary' mirror defined in site's mirrors.yaml. + - ``$SOURCE_CACHE``: Name of or path to (beginning with '\file://') source cache directory. Defaults to 'local-source' mirror defined in site's mirrors.yaml. + - ``$PADDED_LENGTH``: Padded length setting for Spack build cache generation. It should be as long as possible without the build failing. + - ``$KEEP_WEEKLY_BUILD_DIR``: Set to 'YES' to use a persistent directory structure for automated builds. + +------------------------------ +Setting up weekly builds: cron +------------------------------ + +Prior to running, obtain a current copy of the build scripts by cloning the spack-stack repository to the platform of interest (/path/to/spack-stack-auto-build/). + +If /path/to/spack-stack is the local spack-stack root directory containing spack-stack-x.y.z installations, run ``mkdir -p /path/to/spack-stack/weekly_build/logs``. + +In crontab, add: + +.. code-block:: console + + 0 1 * * SUN /path/to/spack-stack-auto-build/util/weekly_build/SpackStackBuildCache_AllSteps.sh $(date +\%y\%m\%d) /path/to/spack-stack/weekly_build acorn > /path/to/spack-stack/weekly_build/logs/cron.$(date +\%y\%m\%d).out 2>&1 From 9ca72522b7b57609f8de61853be2bf68a3748643 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Fri, 15 Nov 2024 09:49:51 -0800 Subject: [PATCH 125/134] add automation docs to index.rst --- doc/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/index.rst b/doc/source/index.rst index e084a1ed1..7a67d65f2 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -19,6 +19,7 @@ Table of contents Utilities KnownIssues MaintainersSection + AutomatedBuilds Documentation Acronyms Glossary From ca88f154ab9b623d6cafd0d5ed7c5af053a60cff Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Fri, 15 Nov 2024 10:08:58 -0800 Subject: [PATCH 126/134] restore ubuntu-ci-containers-x86_64.yaml --- .../ubuntu-ci-containers-x86_64.yaml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/workflows/ubuntu-ci-containers-x86_64.yaml diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml new file mode 100644 index 000000000..b69d0d324 --- /dev/null +++ b/.github/workflows/ubuntu-ci-containers-x86_64.yaml @@ -0,0 +1,111 @@ +name: ubuntu-ci-container-x86_64-build +on: + # Uncomment this to test for PRs (but do not submit) + #pull_request: + # paths-ignore: + # - 'configs/sites/**' + # - 'doc/**' + # - '**.md' + # - '.github/ISSUE_TEMPLATE/*' + # - '.gitignore' + # + schedule: + - cron: '0 8 * * *' + workflow_dispatch: + inputs: + container: + description: 'Container template (input requred, no default value).' + required: true + specs: + description: 'Which specs to add to the template (input requred, no default value).' + required: true + +defaults: + run: + shell: bash + +jobs: + ubuntu-ci-container-x86_64-build: + runs-on: [ubuntu-ci-c6a-x86_64] + + steps: + - name: checkout + uses: actions/checkout@v3 + with: + submodules: true + + # Ensure that ubuntu can run docker + - name: ubuntu-docker + run: | + echo "Checking if user ubuntu can run docker" + if id -nG ubuntu | grep -qw docker; then + echo "User ubuntu already belongs to group docker" + else + echo "Adding user ubuntu to group docker" + sudo gpasswd -a ubuntu docker + newgrp docker + docker run hello-world + fi + + - name: create-ctr + run: | + source ./setup.sh + + # Get day of week to set default container for scheduled builds + DOW=$(date +%u) + # Monday is 1 ... Sunday is 7 + if [[ $DOW == 1 || $DOW == 4 ]]; then + export CONTAINER=${{ inputs.container || 'docker-ubuntu-clang-mpich' }} + export SPECS=${{ inputs.specs || 'jedi-ci' }} + elif [[ $DOW == 2 || $DOW == 5 ]]; then + export CONTAINER=${{ inputs.container || 'docker-ubuntu-gcc-openmpi' }} + export SPECS=${{ inputs.specs || 'jedi-ci' }} + elif [[ $DOW == 3 || $DOW == 6 ]]; then + export CONTAINER=${{ inputs.container || 'docker-ubuntu-intel-impi' }} + export SPECS=${{ inputs.specs || 'jedi-ci' }} + else + # Day 7: The Sabbath of rest (but do some house keeping later on) + exit 0 + fi + + export ENVDIR=$PWD/envs/${CONTAINER} + spack stack create ctr --container ${CONTAINER} --specs ${SPECS} + + cd ${ENVDIR} + spack containerize > Dockerfile + docker build -t ${CONTAINER}-${SPECS} . + + # Report status to JCSDA CI slack channel for nightly runs only + - name: Report Status + if: always() + uses: ravsamhq/notify-slack-action@v1 + env: + SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} + with: + # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url + # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script + # https://github.com/marketplace/actions/notify-slack-action + status: ${{ job.status }} + notify_when: 'success,failure,warnings' + notification_title: '{workflow} has {status_message}' + message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>' + footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }} + # For testing: only notify user Dom + #mention_users: 'U02NLGXF5HV' + #mention_users_when: 'failure,warnings' + # Default: notify channel + mention_groups: '!channel' + mention_groups_when: 'failure,warnings' + + # We can only clean the docker image registry *after* reporting + # the status to the CI slack channel, because the cleanup is + # deleting the docker image for ravsamhq/notify-slack-action@v1 + - name: clean-docker-registry + if: always() + run: | + # Get day of week (only run this on Sunday, which is 7) + DOW=$(date +%u) + if [[ $DOW == 7 ]]; then + echo "Pruning all docker images" + docker system prune -a -f + fi From cf3da0829085318796821972a6c464137a0e1922 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Fri, 15 Nov 2024 10:09:38 -0800 Subject: [PATCH 127/134] Update configs/templates/neptune-dev/spack.yaml Co-authored-by: Dom Heinzeller --- configs/templates/neptune-dev/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 75dfeddb4..9a9e3dfec 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -9,7 +9,7 @@ spack: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - neptune-env +espc ^esmf@8.7.0 - #- neptune-python-env +xnrl ^esmf@8.7.0 + - neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: From 38e59723b5a3ef2f9dc9cbddf2ee03fe99e674cf Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 26 Nov 2024 10:19:26 -0800 Subject: [PATCH 128/134] remove nautilus nightly build yaml's --- .github/workflows/nautilus-nightly-intel.yaml | 49 ------------------- .../workflows/nautilus-nightly-oneapi.yaml | 49 ------------------- 2 files changed, 98 deletions(-) delete mode 100644 .github/workflows/nautilus-nightly-intel.yaml delete mode 100644 .github/workflows/nautilus-nightly-oneapi.yaml diff --git a/.github/workflows/nautilus-nightly-intel.yaml b/.github/workflows/nautilus-nightly-intel.yaml deleted file mode 100644 index 5225263bf..000000000 --- a/.github/workflows/nautilus-nightly-intel.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: nautilus unified-env intel - -on: - workflow_dispatch: - pull_request: - branches: - - develop - types: [opened, synchronize, reopened] - # Is this UTC? - #schedule: - # - cron: '0 8 * * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -env: - BASEDIR: /p/work2/heinzell/spack-stack-nightly/intel - PLATFORM: nautilus - COMPILERS: intel - -jobs: - - nightly-build-intel: - runs-on: ['self-hosted','nautilus-heinzell'] - # Set timeout to 12hrs - timeout-minutes: 720 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - clean: true - - name: 01_DirectorySetup - uses: ./.github/actions/Nightly_01_DirectorySetup - - name: 02_GetSpackStack - uses: ./.github/actions/Nightly_02_GetSpackStack - - name: 03_SetupEnv - uses: ./.github/actions/Nightly_03_SetupEnv - - name: 04_SpackInstall - uses: ./.github/actions/Nightly_04_SpackInstall - - name: 05_BuildCache - uses: ./.github/actions/Nightly_05_BuildCache - # Skip step 6 - app test - for the moment - - name: 07_Cleanup - uses: ./.github/actions/Nightly_07_Cleanup diff --git a/.github/workflows/nautilus-nightly-oneapi.yaml b/.github/workflows/nautilus-nightly-oneapi.yaml deleted file mode 100644 index 4459ceb4d..000000000 --- a/.github/workflows/nautilus-nightly-oneapi.yaml +++ /dev/null @@ -1,49 +0,0 @@ -name: nautilus unified-env oneapi - -on: - workflow_dispatch: - pull_request: - branches: - - develop - types: [opened, synchronize, reopened] - # Is this UTC? - #schedule: - # - cron: '0 8 * * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -env: - BASEDIR: /p/work2/heinzell/spack-stack-nightly/oneapi - PLATFORM: nautilus - COMPILERS: oneapi - -jobs: - - nightly-build-oneapi: - runs-on: ['self-hosted','nautilus-heinzell'] - # Set timeout to 12hrs - timeout-minutes: 720 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - clean: true - - name: 01_DirectorySetup - uses: ./.github/actions/Nightly_01_DirectorySetup - - name: 02_GetSpackStack - uses: ./.github/actions/Nightly_02_GetSpackStack - - name: 03_SetupEnv - uses: ./.github/actions/Nightly_03_SetupEnv - - name: 04_SpackInstall - uses: ./.github/actions/Nightly_04_SpackInstall - - name: 05_BuildCache - uses: ./.github/actions/Nightly_05_BuildCache - # Skip step 6 - app test - for the moment - - name: 07_Cleanup - uses: ./.github/actions/Nightly_07_Cleanup From 82edf52c847c21dae93ef88ea17875d9f7113bc3 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 26 Nov 2024 10:20:25 -0800 Subject: [PATCH 129/134] acorn/compilers.yaml: remove static gcc/etc. flags --- configs/sites/tier1/acorn/compilers.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index 179fcbdbf..7e63ca9c2 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -6,10 +6,7 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: - cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread - cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread - fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + flags: {} operating_system: sles15 modules: - PrgEnv-intel/8.5.0 @@ -30,10 +27,7 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: - cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread - cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread - fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + flags: {} operating_system: sles15 modules: - PrgEnv-intel/8.5.0 From 499ab7531fee107ea27d3cdc7fece46f13131088 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Tue, 26 Nov 2024 23:33:57 +0000 Subject: [PATCH 130/134] drop 'file://' from mirrorpath --- util/weekly_build/04_SpackInstall.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/weekly_build/04_SpackInstall.sh b/util/weekly_build/04_SpackInstall.sh index a40a3f6f1..bddf21f42 100755 --- a/util/weekly_build/04_SpackInstall.sh +++ b/util/weekly_build/04_SpackInstall.sh @@ -30,6 +30,7 @@ for compiler in $COMPILERS; do else mirrorpath=$(spack mirror list | awk "{if (\$1==\"$SOURCE_CACHE\") print \$NF}") fi + mirrorpath=${mirrorpath#file://} spack_wrapper log.fetch mirror create --dependencies \ --directory ${mirrorpath?"Source mirror path could not be determined. Check site's mirrors.yaml."} \ ${PACKAGES_TO_INSTALL:---all} From ae7914a5cb8080029aba6b3528b34572eb58bb66 Mon Sep 17 00:00:00 2001 From: Rick Grubin Date: Wed, 27 Nov 2024 11:49:39 -0600 Subject: [PATCH 131/134] WIP: run one UFS WM test --- util/weekly_build/06_AppTests.sh | 15 +++++++++++++++ util/weekly_build/apptests/test_ufswm.sh | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh index 3764d8a93..73f22141e 100755 --- a/util/weekly_build/06_AppTests.sh +++ b/util/weekly_build/06_AppTests.sh @@ -4,6 +4,21 @@ set -ex if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi +cd $RUNDIR/$RUNID + +set +x +. setup.sh +set -x + +for compiler in $COMPILERS; do + cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} + spack env activate . + spack module lmod refresh -y + spack stack setup-meta-modules +done + +# TODO: test against all compilers +# For now, only Intel (see apptests/test_ufswm.sh) if [ "$TEST_UFSWM" == ON ]; then ./apptests/test_ufswm.sh fi diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh index 0cfcd88ab..1f1b25b5d 100755 --- a/util/weekly_build/apptests/test_ufswm.sh +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -3,3 +3,18 @@ # Do some stuff, and exit non-zero if UFSWM cannot be successfully built and # tested (ideally, have a discernably different error condition if there are # numerical differences) + +cd ${RUNDIR} + +# The following need to be cleaned up for proper variable substitution; this is a one-off test +BRANCH=spack-stack-automation + +git clone -b ${BRANCH} --single-branch --recurse-submodules https://github.com/rickgrubin-noaa/ufs-weather-model.git +cd ufs-weather-model/tests + +# -r ==> rocoto ; not strictly necessary, can run without +./rt.sh -a epic -k -r -n "control_c48 intel" + +rc = $? +return rc + From e262ba27e18437df8fe6400a53b32dbbbe589296 Mon Sep 17 00:00:00 2001 From: Rick Grubin Date: Wed, 4 Dec 2024 16:07:13 -0600 Subject: [PATCH 132/134] clean up testing scripts --- util/weekly_build/06_AppTests.sh | 6 +----- util/weekly_build/apptests/test_ufswm.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh index 73f22141e..3d6f05fc9 100755 --- a/util/weekly_build/06_AppTests.sh +++ b/util/weekly_build/06_AppTests.sh @@ -2,8 +2,6 @@ set -ex -if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi - cd $RUNDIR/$RUNID set +x @@ -17,8 +15,6 @@ for compiler in $COMPILERS; do spack stack setup-meta-modules done -# TODO: test against all compilers -# For now, only Intel (see apptests/test_ufswm.sh) if [ "$TEST_UFSWM" == ON ]; then - ./apptests/test_ufswm.sh + ./apptests/test_ufswm.sh $* fi diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh index 1f1b25b5d..ff2ffd897 100755 --- a/util/weekly_build/apptests/test_ufswm.sh +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -4,17 +4,17 @@ # tested (ideally, have a discernably different error condition if there are # numerical differences) +echo Base directory: ${RUNDIR:?} cd ${RUNDIR} -# The following need to be cleaned up for proper variable substitution; this is a one-off test -BRANCH=spack-stack-automation +UFSWM_BRANCH=${UFSWM_BRANCH:-develop} +UFSWM_URL=${UFSWM_URL:-"https://github.com/ufs-community/ufs-weather-model.git"} -git clone -b ${BRANCH} --single-branch --recurse-submodules https://github.com/rickgrubin-noaa/ufs-weather-model.git +git clone -b ${BRANCH} --single-branch --recurse-submodules ${UFSWM_URL} cd ufs-weather-model/tests -# -r ==> rocoto ; not strictly necessary, can run without -./rt.sh -a epic -k -r -n "control_c48 intel" +# rt.sh will parse arguments passed to it +./rt.sh "${@}" rc = $? return rc - From bd5cf631f63443ac683f6ca5198c6d445a79f13c Mon Sep 17 00:00:00 2001 From: Rick Grubin Date: Wed, 4 Dec 2024 18:04:49 -0600 Subject: [PATCH 133/134] generalize testing, add orion site config --- util/weekly_build/06_AppTests.sh | 2 +- util/weekly_build/apptests/test_ufswm.sh | 5 ++-- util/weekly_build/sites/orion.sh | 29 ++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 util/weekly_build/sites/orion.sh diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh index 3d6f05fc9..301b6ef2d 100755 --- a/util/weekly_build/06_AppTests.sh +++ b/util/weekly_build/06_AppTests.sh @@ -16,5 +16,5 @@ for compiler in $COMPILERS; do done if [ "$TEST_UFSWM" == ON ]; then - ./apptests/test_ufswm.sh $* + ./apptests/test_ufswm.sh fi diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh index ff2ffd897..4caf97daf 100755 --- a/util/weekly_build/apptests/test_ufswm.sh +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -10,11 +10,12 @@ cd ${RUNDIR} UFSWM_BRANCH=${UFSWM_BRANCH:-develop} UFSWM_URL=${UFSWM_URL:-"https://github.com/ufs-community/ufs-weather-model.git"} -git clone -b ${BRANCH} --single-branch --recurse-submodules ${UFSWM_URL} +git clone --single-branch --recurse-submodules ${UFSWM_URL} -b ${UFSWM_BRANCH} cd ufs-weather-model/tests # rt.sh will parse arguments passed to it -./rt.sh "${@}" +RT_ARGS=${RT_ARGS:-"-a ${BATCHACCOUNT:?} -n 'control_c48 intel'"} +./rt.sh $RT_ARGS rc = $? return rc diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh new file mode 100644 index 000000000..a1aabf547 --- /dev/null +++ b/util/weekly_build/sites/orion.sh @@ -0,0 +1,29 @@ + +COMPILERS=${COMPILERS:-"intel"} +TEMPLATES=${TEMPLATES:-"unified-dev"} + +function alert_cmd { + mail -s 'spack-stack weekly build failure' richard.grubin@noaa.gov < <(echo "Weekly spack-stack build failed in $1. Run ID: $RUNID") +} + +module --force purge +umask 0022 + +SPACK_STACK_URL=https://github.com/JCSDA/spack-stack.git +SPACK_STACK_BRANCH=develop + +KEEP_WEEKLY_BUILD_DIR="YES" +PADDED_LENGTH=200 + +# package test / install settings (future use) +# PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" +# PACKAGES_TO_INSTALL="ufs-weather-model-env" + +TEST_UFSWM=ON +UFSWM_BRANCH=develop +UFSWM_URL="https://github.com/ufs-community/ufs-weather-model.git" + +# rt.sh parameters / arguments +BATCHACCOUNT=epic +RT_ARGS="-a ${BATCHACCOUNT} -k -r -n 'control_c48 intel'" + From a95eee0294ec0fc75c21003b01727b3b560dbaa1 Mon Sep 17 00:00:00 2001 From: AlexanderRichert-NOAA Date: Thu, 5 Dec 2024 18:36:58 +0000 Subject: [PATCH 134/134] fix up UWM test logic --- util/weekly_build/06_AppTests.sh | 16 +++++++++++----- util/weekly_build/apptests/test_ufswm.sh | 10 +++++----- util/weekly_build/sites/acorn.sh | 1 + util/weekly_build/sites/orion.sh | 3 +-- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/util/weekly_build/06_AppTests.sh b/util/weekly_build/06_AppTests.sh index 301b6ef2d..a8ee6ffb1 100755 --- a/util/weekly_build/06_AppTests.sh +++ b/util/weekly_build/06_AppTests.sh @@ -2,6 +2,8 @@ set -ex +if [ -z $SETUPDONE ]; then . ShellSetup.sh $* ; fi + cd $RUNDIR/$RUNID set +x @@ -9,12 +11,16 @@ set +x set -x for compiler in $COMPILERS; do - cd $RUNDIR/$RUNID/envs/build-${compiler/@/-} - spack env activate . - spack module lmod refresh -y - spack stack setup-meta-modules + for template in $TEMPLATES; do + envname=build-$template-${compiler/@/-} + envdir=$RUNDIR/$RUNID/envs/$envname + cd $envdir + spack env activate . + spack module lmod refresh -y + spack stack setup-meta-modules + done done if [ "$TEST_UFSWM" == ON ]; then - ./apptests/test_ufswm.sh + . $(dirname $0)/apptests/test_ufswm.sh fi diff --git a/util/weekly_build/apptests/test_ufswm.sh b/util/weekly_build/apptests/test_ufswm.sh index 4caf97daf..030135295 100755 --- a/util/weekly_build/apptests/test_ufswm.sh +++ b/util/weekly_build/apptests/test_ufswm.sh @@ -4,18 +4,18 @@ # tested (ideally, have a discernably different error condition if there are # numerical differences) -echo Base directory: ${RUNDIR:?} -cd ${RUNDIR} +echo "Current directory, in which ufs-weather-model will be cloned: $PWD" UFSWM_BRANCH=${UFSWM_BRANCH:-develop} UFSWM_URL=${UFSWM_URL:-"https://github.com/ufs-community/ufs-weather-model.git"} -git clone --single-branch --recurse-submodules ${UFSWM_URL} -b ${UFSWM_BRANCH} +if [ ! -d ufs-weather-model ]; then + git clone --recurse-submodules --single-branch --depth 1 --shallow-submodules ${UFSWM_URL} -b ${UFSWM_BRANCH} +fi cd ufs-weather-model/tests # rt.sh will parse arguments passed to it -RT_ARGS=${RT_ARGS:-"-a ${BATCHACCOUNT:?} -n 'control_c48 intel'"} -./rt.sh $RT_ARGS +./rt.sh $RT_ARGS -a ${BATCHACCOUNT:?} -n 'control_c48 intel' rc = $? return rc diff --git a/util/weekly_build/sites/acorn.sh b/util/weekly_build/sites/acorn.sh index f20f7da29..45420c7cf 100644 --- a/util/weekly_build/sites/acorn.sh +++ b/util/weekly_build/sites/acorn.sh @@ -26,3 +26,4 @@ PACKAGES_TO_TEST="libpng libaec jasper w3emc g2c" PACKAGES_TO_INSTALL="ufs-weather-model-env global-workflow-env gsi-env madis" PADDED_LENGTH=140 TEST_UFSWM=ON +BATCHACCOUNT=NCEPLIBS-DEV diff --git a/util/weekly_build/sites/orion.sh b/util/weekly_build/sites/orion.sh index a1aabf547..9f202a9f5 100644 --- a/util/weekly_build/sites/orion.sh +++ b/util/weekly_build/sites/orion.sh @@ -25,5 +25,4 @@ UFSWM_URL="https://github.com/ufs-community/ufs-weather-model.git" # rt.sh parameters / arguments BATCHACCOUNT=epic -RT_ARGS="-a ${BATCHACCOUNT} -k -r -n 'control_c48 intel'" - +RT_ARGS="-k -r"