Merge branch 'main' into openfoam

.github/workflows/scorecards.yml

@@ -22,6 +22,7 @@ permissions: read-all
 
 jobs:
   analysis:
+    if: github.repository == 'EESSI/test-suite'   # Prevent running on forks
     name: Scorecards analysis
     runs-on: ubuntu-latest
     permissions:

.github/workflows/test.yml

@@ -1,5 +1,5 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
-name: Tests for EESSI test suite, using EESSI pilot repo
+name: Tests for EESSI test suite, using EESSI production repo
 on: [push, pull_request, workflow_dispatch]
 permissions: read-all
 jobs:
@@ -9,39 +9,23 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-        - "2021.12"
+        - '2023.06'
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
           with:
             persist-credentials: false
 
-        - name: Mount EESSI CernVM-FS pilot repository
-          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+        - name: Mount EESSI CernVM-FS production repository
+          uses: eessi/github-action-eessi@e1f8f20638ea417a18d23ab29443ee34794ff900 # v3.1.0
           with:
-              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
-              cvmfs_http_proxy: DIRECT
-              cvmfs_repositories: pilot.eessi-hpc.org
+              eessi_stack_version: ${{matrix.EESSI_VERSION}}
 
         - name: Run test suite
           run: |
-            source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+            source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
 
-            # install latest version of EasyBuild, to install ReFrame with it,
-            # since that includes the ReFrame test library (hpctestlib) that we rely on
-            python3 -m venv venv
-            source venv/bin/activate
-            pip3 install easybuild
-            eb --version
-            export EASYBUILD_PREFIX=$HOME/easybuild
-            # need to force module generation with --module-only --force because 'pip check' fails
-            # in EESSI pilot 2021.12, see https://github.com/EESSI/compatibility-layer/issues/152
-            eb ReFrame-4.3.3.eb || eb ReFrame-4.3.3.eb --module-only --force
-
-            # load ReFrame
-            module use $HOME/easybuild/modules/all
-
-            module load ReFrame/4.3.3
+            module load ReFrame
             reframe --version
 
             # configure ReFrame (cfr. https://reframe-hpc.readthedocs.io/en/stable/manpage.html#environment)

CI/README.md

@@ -28,7 +28,8 @@ It should define:
 - `REFRAME_VERSION` (mandatory): the version of ReFrame you'd like to use to drive the EESSI test suite in the CI pipeline.
 - `REFRAME_URL` (optional): the URL that will be used to `git clone` the ReFrame repository (in order to provide the `hpctestlib`). Typically this points to the official repository, but you may want to use another URL from a fork for development purposes. Default: `https://github.com/reframe-hpc/reframe.git`.
 - `REFRAME_BRANCH` (optional): the branch name to be cloned for the ReFrame repository (in order to provide the `hpctestlib`). Typically this points to the branch corresponding with `${REFRAME_VERSION}`, unless you want to run from a feature branch for development purposes. Default: `v${REFRAME_VERSION}`.
-- `EESSI_VERSION` (mandatory): the version of the EESSI software stack you would like to be loaded & tested in the CI pipeline.
+- `EESSI_CVMFS_REPO` (optional): the prefix for the CVMFS repository to use, e.g. `/cvmfs/software.eessi.io`
+- `EESSI_VERSION` (optional): the version of the EESSI software stack you would like to be loaded & tested in the CI pipeline.
 - `EESSI_TESTSUITE_URL` (optional): the URL that will be used to `git clone` the `EESSI/test-suite` repository. Typically this points to the official repository, but you may want to use another URL from a fork for development purposes. Default: `https://github.com/EESSI/test-suite.git`.
 - `EESSI_TESTSUITE_VERSION` (optional): the version of the EESSI test-suite repository you want to use in the CI pipeline. Default: latest release.
 - `RFM_CONFIG_FILES` (optional): the location of the ReFrame configuration file to be used for this system. Default: `${TEMPDIR}/test-suite/config/${EESSI_CI_SYSTEM_NAME}.py`.
@@ -44,6 +45,13 @@ echo "0 0 * * SUN EESSI_CI_SYSTEM_NAME=aws_citc ${HOME}/test-suite/CI/run_refram
 ```
 Would create a cronjob running weekly on Sundays. See the crontab manual for other schedules.
 
+Note that you can overwrite the settings in the ci_config.sh by setting environment variables in the crontab. E.g. the following crontab file would run single node and 2-node tests daily, and 1, 2, 4, 8, and 16-node tests weekly (on Sundays):
+```
+# crontab file
+0 0 * * * EESSI_CI_SYSTEM_NAME=aws_mc REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" ${HOME}/test-suite/CI/run_reframe_wrapper.sh
+0 0 * * SUN EESSI_CI_SYSTEM_NAME=aws_mc REFRAME_ARGS="--tag CI --tag 1_node|2_nodes|4_nodes|8_nodes|16_nodes" ${HOME}/test-suite/CI/run_reframe_wrapper.sh
+```
+
 ## Output of the CI pipeline
 The whole point of the `run_reframe_wrapper.sh` script is to easily get the stdout and stderr from your `run_reframe.sh` in a time-stamped logfile. By default, these are stored in `${HOME}/EESSI_CI_LOGS`. This can be changed by setting the environment variable `EESSI_CI_LOGDIR`. Again, you'd have to set this when creating your `crontab` file, e.g.
 ```

CI/aws_mc/ci_config.sh

@@ -1,7 +1,4 @@
 # Configurable items
-REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
-REFRAME_VERSION=4.4.1  # ReFrame version that will be pip-installed to drive the test suite
-# Latest release does not contain the `aws_mc.py` ReFrame config yet
-# The custom EESSI_TESTSUITE_URL and EESSI_TESTSUITE_BRANCH can be removed in a follow-up PR
-EESSI_TESTSUITE_URL='https://github.com/casparvl/test-suite.git'
-EESSI_TESTSUITE_BRANCH='CI'
+if [ -z "${REFRAME_ARGS}" ]; then
+   REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
+fi

CI/it4i_karolina/ci_config.sh

@@ -1,7 +1,4 @@
 # Configurable items
-REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
-REFRAME_VERSION=4.4.1  # ReFrame version that will be pip-installed to drive the test suite
-# Latest release does not contain the `aws_mc.py` ReFrame config yet
-# The custom EESSI_TESTSUITE_URL and EESSI_TESTSUITE_BRANCH can be removed in a follow-up PR
-EESSI_TESTSUITE_URL='https://github.com/casparvl/test-suite.git'
-EESSI_TESTSUITE_BRANCH='CI'
+if [ -z "${REFRAME_ARGS}" ]; then
+    REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
+fi

CI/izum_vega/ci_config.sh

@@ -1,3 +1,4 @@
 # Configurable items
-REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
-REFRAME_VERSION=4.4.1  # ReFrame version that will be pip-installed to drive the test suite
+if [ -z "${REFRAME_ARGS}" ]; then
+    REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
+fi

CI/run_reframe.sh

@@ -4,32 +4,40 @@
 # Setup instructions: make sure you have your github access key configured in your .ssh/config
 # i.e. configure an entry with HostName github.com and IdentityFile pointing to the ssh key registered with Github
 
+# Print on which host this CI is running
+echo "Running CI on host $(hostname)"
+
 # Get directory of the current script
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
+echo $SCRIPT_DIR
 # Check if EESSI_CI_SYSTEM_NAME is defined
 if [ -z "${EESSI_CI_SYSTEM_NAME}" ]; then 
     echo "You have to define the EESSI_CI_SYSTEM_NAME environment variable in order to run the EESSI test suite CI" > /dev/stderr
+    echo "Valid EESSI_CI_SYSTEM_NAME's are:"
+    echo "$(find $SCRIPT_DIR -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)"
     exit 1
 fi
 
 # Check if CI_CONFIG file file exists
 CI_CONFIG="${SCRIPT_DIR}/${EESSI_CI_SYSTEM_NAME}/ci_config.sh"
 if [ ! -f "${CI_CONFIG}" ]; then
-    echo "File ${CI_CONFIG} does not exist. Please check your RFM_CI_SYSTEM_NAME (${EESSI_CI_SYSTEM_NAME}) and make sure the directory in which the current script resides (${SCRIPT_DIR}) contains a subdirectory with that name, and a CI configuration file (ci_config.sh) inside". > /dev/stderr
+    echo "File ${CI_CONFIG} does not exist. Please check your EESSI_CI_SYSTEM_NAME (${EESSI_CI_SYSTEM_NAME}) and make sure the directory in which the current script resides (${SCRIPT_DIR}) contains a subdirectory with that name, and a CI configuration file (ci_config.sh) inside". > /dev/stderr
     exit 1
 fi
 
 # Set the CI configuration for this system
 source "${CI_CONFIG}"
 
-# Set default configuration
+# Set default configuration, but let anything set by CI_CONFIG take priority
 if [ -z "${TEMPDIR}" ]; then
     TEMPDIR=$(mktemp --directory --tmpdir=/tmp  -t rfm.XXXXXXXXXX)
 fi
 if [ -z "${REFRAME_ARGS}" ]; then
     REFRAME_ARGS="--tag CI --tag 1_node"
 fi
+if [ -z "${REFRAME_VERSION}"]; then
+    REFRAME_VERSION=4.5.1
+fi
 if [ -z "${REFRAME_URL}" ]; then
     REFRAME_URL='https://github.com/reframe-hpc/reframe.git'
 fi
@@ -40,10 +48,13 @@ if [ -z "${EESSI_TESTSUITE_URL}" ]; then
     EESSI_TESTSUITE_URL='https://github.com/EESSI/test-suite.git'
 fi
 if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then
-    EESSI_TESTSUITE_BRANCH='v0.1.0'
+    EESSI_TESTSUITE_BRANCH='v0.3.2'
+fi
+if [ -z "${EESSI_CVMFS_REPO}" ]; then
+    export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io
 fi
 if [ -z "${EESSI_VERSION}" ]; then
-    EESSI_VERSION='latest'
+    export EESSI_VERSION=2023.06
 fi
 if [ -z "${RFM_CONFIG_FILES}" ]; then
     export RFM_CONFIG_FILES="${TEMPDIR}/test-suite/config/${EESSI_CI_SYSTEM_NAME}.py"
@@ -74,11 +85,7 @@ export PYTHONPATH="${PYTHONPATH}":"${TEMPDIR}"/test-suite/
 
 # Start the EESSI environment
 unset MODULEPATH
-if [ "${EESSI_VERSION}" = 'latest' ]; then
-    eessi_init_path=/cvmfs/pilot.eessi-hpc.org/latest/init/bash
-else
-    eessi_init_path=/cvmfs/pilot.eessi-hpc.org/versions/"${EESSI_VERSION}"/init/bash
-fi
+eessi_init_path="${EESSI_CVMFS_REPO}"/versions/"${EESSI_VERSION}"/init/bash
 source "${eessi_init_path}"
 
 # Needed in order to make sure the reframe from our TEMPDIR is first on the PATH,

CI/surf_snellius/ci_config.sh

@@ -1,3 +1,4 @@
 # Configurable items
-REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
-REFRAME_VERSION=4.4.1  # ReFrame version that will be pip-installed to drive the test suite
+if [ -z "${REFRAME_ARGS}" ]; then
+    REFRAME_ARGS="--tag CI --tag 1_node|2_nodes"
+fi

README.md

@@ -39,6 +39,15 @@ origin  [email protected]:EESSI/test-suite.git (fetch)
 origin  [email protected]:EESSI/test-suite.git (push)
 ```
 
+#### Option 1: Creating a branch from the PR directly
+
+```bash
+git fetch origin pull/ID/head:BRANCH_NAME
+```
+where `ID` is the number of the pull request, and `BRANCH_NAME` is the name of the local branch (you can pick this yourself).
+
+#### Option 2: Creating a branch tracking the feature branch
+
 You can add a fork to your local clone by adding a new remote. Pick a name for
 the remote that you find easy to recognize. E.g. to add the fork
 https://github.com/casparvl/test-suite and give it the (local) name `casparvl`,
@@ -89,3 +98,18 @@ is that it is easy to pull in updates from a feature branch using `git pull`.
 You can also push back changes to the feature branch directly, but note that
 you are pushing to the Github fork of another Github user, so _make sure they
 are ok with that_ before doing so!
+
+## Release management
+
+When a release of the EESSI test suite is made, the following things must be taken care of:
+
+- Version bump: in both `pyproject.toml` and `setup.cfg`;
+- Version bump the default `EESSI_TESTSUITE_BRANCH` in `CI/run_reframe.sh`;
+- Release notes: in `RELEASE_NOTES` + in GitHub release (cfr. https://github.com/EESSI/test-suite/releases/tag/v0.2.0);
+- Tag release on GitHub + publish release (incl. release notes);
+- Publishing release to PyPI:
+  ```
+  # example for version 0.2.0
+  python setup.py sdist
+  twine upload dist/eessi_testsuite-0.2.0.tar.gz
+  ```

RELEASE_NOTES

@@ -1,7 +1,51 @@
 This file contains a description of the major changes to the EESSI test suite.
 For more detailed information, please see the git log.
 
-v0.2.0 (7 march 2024)
+v0.3.2 (29 June 2024)
+---------------------
+
+This is a bugfix release of the EESSI test-suite
+
+It includes:
+
+- Add config for Deucalion (#162)
+
+v0.3.1 (28 June 2024)
+---------------------
+
+This is a bugfix release of the EESSI test-suite
+
+It includes:
+
+- Correct required memory per node to MiB in ESPResSo test (#158)
+- Change behavior for assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) on hyperthreading-enabled systems (#160)
+- Use compact process binding in ESPResSo test (#160)
+
+v0.3.0 (27 June 2024)
+---------------------
+
+This is a minor release of the EESSI test-suite
+
+It includes:
+
+* Update config AWS MC cluster to use `software.eessi.io` (#126)
+* Add test for QuantumESPRESSO (pw.x) (#128)
+* Fix compact process binding for OpenMPI mpirun (#137)
+* Use compact process binding for GROMACS (#139)
+* Rename scale tags 1_cpn_2_nodes and 1_cpn_4_nodes (#140)
+* Set SRUN_CPUS_PER_TASK for srun launcher (#141)
+* Fix for "Failed to modify UD QP to INIT on mlx5_0" on Karolina CI runs (#142)
+* Reduce the iteration count to make the OSU tests run faster, especially on slower interconnects (#143)
+* Add test for ESPResSo (P3M) (#144)
+* Use software.eessi.io repo in CI (#146)
+* Add notes on release management to README (#148)
+* Fix memory_per_node for Hortense (#151)
+* Use MiB units for memory per node (#152)
+* Added / updated memory for various systems in MiB units (#153)
+* Add additional test for ESPRESSO (LJ) (#155)
+* Bump default version used in CI (#157)
+
+v0.2.0 (7 March 2024)
 ---------------------
 
 This is a minor release of the EESSI test-suite

config/aws_mc.py

@@ -105,6 +105,11 @@
         # steps inherit environment. It doesn't hurt to define this even if srun is not used
         'export SLURM_EXPORT_ENV=ALL'
     ],
+    'extras': {
+        # Node types have somewhat varying amounts of memory, but we'll make it easy on ourselves
+        # All should _at least_ have this amount (30GB * 1E9 / (1024*1024) = 28610 MiB)
+        'mem_per_node': 28610
+    },
 }
 for system in site_configuration['systems']:
     for partition in system['partitions']:

config/github_actions.py

@@ -18,14 +18,23 @@
                     'launcher': 'local',
                     'environs': ['default'],
                     'features': [FEATURES[CPU]] + list(SCALES.keys()),
-                    'processor': {'num_cpus': 2},
+                    'processor': {
+                        'num_cpus': 2,
+                        'num_cpus_per_core': 1,
+                    },
                     'resources': [
                         {
                             'name': 'memory',
                             'options': ['--mem={size}'],
                         }
                     ],
-                    'max_jobs': 1
+                    'max_jobs': 1,
+                    'extras': {
+                        # Make sure to round down, otherwise a job might ask for more mem than is available
+                        # per node
+                        # This is a fictional amount, GH actions probably has less, but only does --dry-run
+                        'mem_per_node': 30 * 1024  # in MiB
+                    },
                 }
             ]
         }

config/it4i_karolina.py

@@ -44,6 +44,12 @@
                         # Avoid https://github.com/EESSI/software-layer/issues/136
                         # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1)
                         'export OMPI_MCA_pml=ucx',
+                        # Work around "Failed to modify UD QP to INIT on mlx5_0: Operation not permitted" issue
+                        # until we can resolve this through an LMOD hook in host_injections.
+                        # (then these OMPI_MCA_btl & mtl can be removed again)
+                        # See https://github.com/EESSI/software-layer/issues/456#issuecomment-2107755266
+                        'export OMPI_MCA_mtl="^ofi"',
+                        'export OMPI_MCA_btl="^ofi"',
                     ],
                     'launcher': 'mpirun',
                     # Use --export=None to avoid that login environment is passed down to submitted jobs
@@ -53,6 +59,11 @@
                     'features': [
                         FEATURES[CPU],
                     ] + list(SCALES.keys()),
+                    'extras': {
+                        # Make sure to round down, otherwise a job might ask for more mem than is available
+                        # per node
+                        'mem_per_node': 235520  # in MiB
+                    },
                     'descr': 'CPU Universal Compute Nodes, see https://docs.it4i.cz/karolina/hardware-overview/'
                 },
                 # We don't have GPU budget on Karolina at this time