From 032cfb2dfb354eed98d62192fcefb1027eaa549d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 12:14:08 +0100 Subject: [PATCH 001/119] Updated to testsuite branch 0.2.0 for CI configs. Also, specify a default ReFrame version in run_reframe.sh --- CI/aws_mc/ci_config.sh | 5 ----- CI/it4i_karolina/ci_config.sh | 5 ----- CI/izum_vega/ci_config.sh | 1 - CI/run_reframe.sh | 7 +++++-- CI/surf_snellius/ci_config.sh | 1 - 5 files changed, 5 insertions(+), 14 deletions(-) diff --git a/CI/aws_mc/ci_config.sh b/CI/aws_mc/ci_config.sh index d7975420..79bb174e 100644 --- a/CI/aws_mc/ci_config.sh +++ b/CI/aws_mc/ci_config.sh @@ -1,7 +1,2 @@ # Configurable items REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" -REFRAME_VERSION=4.4.1 # ReFrame version that will be pip-installed to drive the test suite -# Latest release does not contain the `aws_mc.py` ReFrame config yet -# The custom EESSI_TESTSUITE_URL and EESSI_TESTSUITE_BRANCH can be removed in a follow-up PR -EESSI_TESTSUITE_URL='https://github.com/casparvl/test-suite.git' -EESSI_TESTSUITE_BRANCH='CI' diff --git a/CI/it4i_karolina/ci_config.sh b/CI/it4i_karolina/ci_config.sh index d7975420..79bb174e 100644 --- a/CI/it4i_karolina/ci_config.sh +++ b/CI/it4i_karolina/ci_config.sh @@ -1,7 +1,2 @@ # Configurable items REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" -REFRAME_VERSION=4.4.1 # ReFrame version that will be pip-installed to drive the test suite -# Latest release does not contain the `aws_mc.py` ReFrame config yet -# The custom EESSI_TESTSUITE_URL and EESSI_TESTSUITE_BRANCH can be removed in a follow-up PR -EESSI_TESTSUITE_URL='https://github.com/casparvl/test-suite.git' -EESSI_TESTSUITE_BRANCH='CI' diff --git a/CI/izum_vega/ci_config.sh b/CI/izum_vega/ci_config.sh index 4eac6b3c..79bb174e 100644 --- a/CI/izum_vega/ci_config.sh +++ b/CI/izum_vega/ci_config.sh @@ -1,3 +1,2 @@ # Configurable items REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" -REFRAME_VERSION=4.4.1 # ReFrame version that will be pip-installed to drive the test suite diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 534c6651..578813f2 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -23,13 +23,16 @@ fi # Set the CI configuration for this system source "${CI_CONFIG}" -# Set default configuration +# Set default configuration, but let anything set by CI_CONFIG take priority if [ -z "${TEMPDIR}" ]; then TEMPDIR=$(mktemp --directory --tmpdir=/tmp -t rfm.XXXXXXXXXX) fi if [ -z "${REFRAME_ARGS}" ]; then REFRAME_ARGS="--tag CI --tag 1_node" fi +if [ -z "${REFRAME_VERSION}"]; then + REFRAME_VERSION=4.5.1 +fi if [ -z "${REFRAME_URL}" ]; then REFRAME_URL='https://github.com/reframe-hpc/reframe.git' fi @@ -40,7 +43,7 @@ if [ -z "${EESSI_TESTSUITE_URL}" ]; then EESSI_TESTSUITE_URL='https://github.com/EESSI/test-suite.git' fi if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then - EESSI_TESTSUITE_BRANCH='v0.1.0' + EESSI_TESTSUITE_BRANCH='v0.2.0' fi if [ -z "${EESSI_VERSION}" ]; then EESSI_VERSION='latest' diff --git a/CI/surf_snellius/ci_config.sh b/CI/surf_snellius/ci_config.sh index 4eac6b3c..79bb174e 100644 --- a/CI/surf_snellius/ci_config.sh +++ b/CI/surf_snellius/ci_config.sh @@ -1,3 +1,2 @@ # Configurable items REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" -REFRAME_VERSION=4.4.1 # ReFrame version that will be pip-installed to drive the test suite From aa4eaaa56edca17de46db260cbf2d37de94d845b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 12:14:42 +0100 Subject: [PATCH 002/119] This system doesnt exist anymore --- CI/aws_citc/ci_config.sh | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 CI/aws_citc/ci_config.sh diff --git a/CI/aws_citc/ci_config.sh b/CI/aws_citc/ci_config.sh deleted file mode 100644 index 4eac6b3c..00000000 --- a/CI/aws_citc/ci_config.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Configurable items -REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" -REFRAME_VERSION=4.4.1 # ReFrame version that will be pip-installed to drive the test suite From 1817e1e0feeb9b67b6958d93d0091a435f6647ec Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 12:22:22 +0100 Subject: [PATCH 003/119] Print hostname so that it is easier to find which host has the cronjob... --- CI/run_reframe.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 578813f2..25b72cbd 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -4,6 +4,9 @@ # Setup instructions: make sure you have your github access key configured in your .ssh/config # i.e. configure an entry with HostName github.com and IdentityFile pointing to the ssh key registered with Github +# Print on which host this CI is running +echo "Running CI on host $(hostname)" + # Get directory of the current script SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) From 9f880a87020ce66700dfa724c253860c43df16ce Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 13:59:34 +0100 Subject: [PATCH 004/119] Use the production repo in the daily runs --- CI/run_reframe.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 25b72cbd..db777753 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -63,6 +63,12 @@ fi if [ -z "${RFM_PREFIX}" ]; then export RFM_PREFIX="${HOME}/reframe_CI_runs" fi +if [ -z "${EESSI_CVMFS_REPO}" ]; then + export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io +fi +if [ -z "${EESSI_VERSION}" ]; then + export EESSI_VERSION=2023.06 +fi # Create virtualenv for ReFrame using system python python3 -m venv "${TEMPDIR}"/reframe_venv @@ -80,11 +86,7 @@ export PYTHONPATH="${PYTHONPATH}":"${TEMPDIR}"/test-suite/ # Start the EESSI environment unset MODULEPATH -if [ "${EESSI_VERSION}" = 'latest' ]; then - eessi_init_path=/cvmfs/pilot.eessi-hpc.org/latest/init/bash -else - eessi_init_path=/cvmfs/pilot.eessi-hpc.org/versions/"${EESSI_VERSION}"/init/bash -fi +eessi_init_path="${EESSI_CVMFS_REPO}"/versions/"${EESSI_VERSION}"/init/bash source "${eessi_init_path}" # Needed in order to make sure the reframe from our TEMPDIR is first on the PATH, From a7b361b7f86521efc88015abd47b451cd1217275 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 14:42:53 +0100 Subject: [PATCH 005/119] Remove duplicate definition of EESSI_VERSION --- CI/run_reframe.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index db777753..9e52bed6 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -48,8 +48,11 @@ fi if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then EESSI_TESTSUITE_BRANCH='v0.2.0' fi +if [ -z "${EESSI_CVMFS_REPO}" ]; then + export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io +fi if [ -z "${EESSI_VERSION}" ]; then - EESSI_VERSION='latest' + export EESSI_VERSION=2023.06 fi if [ -z "${RFM_CONFIG_FILES}" ]; then export RFM_CONFIG_FILES="${TEMPDIR}/test-suite/config/${EESSI_CI_SYSTEM_NAME}.py" @@ -63,12 +66,6 @@ fi if [ -z "${RFM_PREFIX}" ]; then export RFM_PREFIX="${HOME}/reframe_CI_runs" fi -if [ -z "${EESSI_CVMFS_REPO}" ]; then - export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io -fi -if [ -z "${EESSI_VERSION}" ]; then - export EESSI_VERSION=2023.06 -fi # Create virtualenv for ReFrame using system python python3 -m venv "${TEMPDIR}"/reframe_venv From d36a0de914a1252ea01d4bb07aa8b84ae5af535b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 15:37:39 +0100 Subject: [PATCH 006/119] Make exception for AWS_MC cluster and run pilot, since software.eessi.io is not deployed on the login nodes there --- CI/aws_mc/ci_config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CI/aws_mc/ci_config.sh b/CI/aws_mc/ci_config.sh index 79bb174e..0f22707c 100644 --- a/CI/aws_mc/ci_config.sh +++ b/CI/aws_mc/ci_config.sh @@ -1,2 +1,4 @@ # Configurable items REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +# For now, software.eessi.io is not yet deployed on login nodes of the AWS MC cluster +EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org" From f88cf171e67075c89459094e768c51de5784bf08 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 16:07:22 +0100 Subject: [PATCH 007/119] Make sure to not overwrite when its already set --- CI/surf_snellius/ci_config.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/surf_snellius/ci_config.sh b/CI/surf_snellius/ci_config.sh index 79bb174e..4bd95bef 100644 --- a/CI/surf_snellius/ci_config.sh +++ b/CI/surf_snellius/ci_config.sh @@ -1,2 +1,4 @@ # Configurable items -REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +if [ -z "${REFRAME_ARGS}" ]; then + REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +fi From 71a7e0bfbc2cf2a741649f097c6e7a05438884c6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 16:10:39 +0100 Subject: [PATCH 008/119] Make sure things set in the environment take priority over the ci_config.sh settings --- CI/aws_mc/ci_config.sh | 8 ++++++-- CI/it4i_karolina/ci_config.sh | 4 +++- CI/izum_vega/ci_config.sh | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CI/aws_mc/ci_config.sh b/CI/aws_mc/ci_config.sh index 0f22707c..1ea9d001 100644 --- a/CI/aws_mc/ci_config.sh +++ b/CI/aws_mc/ci_config.sh @@ -1,4 +1,8 @@ # Configurable items -REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +if [ -z "${REFRAME_ARGS}" ]; then + REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +fi # For now, software.eessi.io is not yet deployed on login nodes of the AWS MC cluster -EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org" +if [ -z "${EESSI_CVMFS_REPO}" ]; then + EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org" +fi diff --git a/CI/it4i_karolina/ci_config.sh b/CI/it4i_karolina/ci_config.sh index 79bb174e..4bd95bef 100644 --- a/CI/it4i_karolina/ci_config.sh +++ b/CI/it4i_karolina/ci_config.sh @@ -1,2 +1,4 @@ # Configurable items -REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +if [ -z "${REFRAME_ARGS}" ]; then + REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +fi diff --git a/CI/izum_vega/ci_config.sh b/CI/izum_vega/ci_config.sh index 79bb174e..4bd95bef 100644 --- a/CI/izum_vega/ci_config.sh +++ b/CI/izum_vega/ci_config.sh @@ -1,2 +1,4 @@ # Configurable items -REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +if [ -z "${REFRAME_ARGS}" ]; then + REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" +fi From cb0243cffec50b581cbe8c7b7bc8223432dfe9c1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 16:15:09 +0100 Subject: [PATCH 009/119] Document EESSI_CVMFS_REPO config item. Also, document possibility to overwrite config items through environment --- CI/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CI/README.md b/CI/README.md index 65e6d1e8..161c4a48 100644 --- a/CI/README.md +++ b/CI/README.md @@ -28,7 +28,8 @@ It should define: - `REFRAME_VERSION` (mandatory): the version of ReFrame you'd like to use to drive the EESSI test suite in the CI pipeline. - `REFRAME_URL` (optional): the URL that will be used to `git clone` the ReFrame repository (in order to provide the `hpctestlib`). Typically this points to the official repository, but you may want to use another URL from a fork for development purposes. Default: `https://github.com/reframe-hpc/reframe.git`. - `REFRAME_BRANCH` (optional): the branch name to be cloned for the ReFrame repository (in order to provide the `hpctestlib`). Typically this points to the branch corresponding with `${REFRAME_VERSION}`, unless you want to run from a feature branch for development purposes. Default: `v${REFRAME_VERSION}`. -- `EESSI_VERSION` (mandatory): the version of the EESSI software stack you would like to be loaded & tested in the CI pipeline. +- `EESSI_CVMFS_REPO` (optional): the prefix for the CVMFS repository to use, e.g. `/cvmfs/software.eessi.io` +- `EESSI_VERSION` (optional): the version of the EESSI software stack you would like to be loaded & tested in the CI pipeline. - `EESSI_TESTSUITE_URL` (optional): the URL that will be used to `git clone` the `EESSI/test-suite` repository. Typically this points to the official repository, but you may want to use another URL from a fork for development purposes. Default: `https://github.com/EESSI/test-suite.git`. - `EESSI_TESTSUITE_VERSION` (optional): the version of the EESSI test-suite repository you want to use in the CI pipeline. Default: latest release. - `RFM_CONFIG_FILES` (optional): the location of the ReFrame configuration file to be used for this system. Default: `${TEMPDIR}/test-suite/config/${EESSI_CI_SYSTEM_NAME}.py`. @@ -44,6 +45,13 @@ echo "0 0 * * SUN EESSI_CI_SYSTEM_NAME=aws_citc ${HOME}/test-suite/CI/run_refram ``` Would create a cronjob running weekly on Sundays. See the crontab manual for other schedules. +Note that you can overwrite the settings in the ci_config.sh by setting environment variables in the crontab. E.g. the following crontab file would run single node and 2-node tests daily, and 1, 2, 4, 8, and 16-node tests weekly (on Sundays): +``` +# crontab file +0 0 * * * EESSI_CI_SYSTEM_NAME=aws_mc REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" ${HOME}/test-suite/CI/run_reframe_wrapper.sh +0 0 * * SUN EESSI_CI_SYSTEM_NAME=aws_mc REFRAME_ARGS="--tag CI --tag 1_node|2_nodes|4_nodes|8_nodes|16_nodes" ${HOME}/test-suite/CI/run_reframe_wrapper.sh +``` + ## Output of the CI pipeline The whole point of the `run_reframe_wrapper.sh` script is to easily get the stdout and stderr from your `run_reframe.sh` in a time-stamped logfile. By default, these are stored in `${HOME}/EESSI_CI_LOGS`. This can be changed by setting the environment variable `EESSI_CI_LOGDIR`. Again, you'd have to set this when creating your `crontab` file, e.g. ``` From 9523e3d98eadced7e4f43448a45c88785a6d2022 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 17:31:32 +0100 Subject: [PATCH 010/119] More clear error if you don't set EESSI_CI_SYSTEM_NAME --- CI/run_reframe.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 9e52bed6..2068f03f 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -9,17 +9,19 @@ echo "Running CI on host $(hostname)" # Get directory of the current script SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - +echo $SCRIPT_DIR # Check if EESSI_CI_SYSTEM_NAME is defined if [ -z "${EESSI_CI_SYSTEM_NAME}" ]; then echo "You have to define the EESSI_CI_SYSTEM_NAME environment variable in order to run the EESSI test suite CI" > /dev/stderr + echo "Valid EESSI_CI_SYSTEM_NAME's are:" + echo "$(find $SCRIPT_DIR -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)" exit 1 fi # Check if CI_CONFIG file file exists CI_CONFIG="${SCRIPT_DIR}/${EESSI_CI_SYSTEM_NAME}/ci_config.sh" if [ ! -f "${CI_CONFIG}" ]; then - echo "File ${CI_CONFIG} does not exist. Please check your RFM_CI_SYSTEM_NAME (${EESSI_CI_SYSTEM_NAME}) and make sure the directory in which the current script resides (${SCRIPT_DIR}) contains a subdirectory with that name, and a CI configuration file (ci_config.sh) inside". > /dev/stderr + echo "File ${CI_CONFIG} does not exist. Please check your EESSI_CI_SYSTEM_NAME (${EESSI_CI_SYSTEM_NAME}) and make sure the directory in which the current script resides (${SCRIPT_DIR}) contains a subdirectory with that name, and a CI configuration file (ci_config.sh) inside". > /dev/stderr exit 1 fi From 0cea03fea8ec81ac60fe85bc068a439340d67030 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 12 Mar 2024 21:44:37 +0100 Subject: [PATCH 011/119] software.eessi.io is now also present on the AWS MC cluster --- CI/aws_mc/ci_config.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CI/aws_mc/ci_config.sh b/CI/aws_mc/ci_config.sh index 1ea9d001..953f1c4c 100644 --- a/CI/aws_mc/ci_config.sh +++ b/CI/aws_mc/ci_config.sh @@ -2,7 +2,3 @@ if [ -z "${REFRAME_ARGS}" ]; then REFRAME_ARGS="--tag CI --tag 1_node|2_nodes" fi -# For now, software.eessi.io is not yet deployed on login nodes of the AWS MC cluster -if [ -z "${EESSI_CVMFS_REPO}" ]; then - EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org" -fi From 82e6812d5cf0c3126e344465e27264b6f7c78acb Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 19 Mar 2024 11:24:39 +0100 Subject: [PATCH 012/119] Added test for QE `pw.x` --- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 eessi/testsuite/tests/apps/QuantumESPRESSO.py diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py new file mode 100644 index 00000000..a7bb6dd4 --- /dev/null +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -0,0 +1,93 @@ +""" +This module tests the binary 'pw.x' in available modules containing substring 'QuantumESPRESSO'. +Test input files are defined in the ReFrame test library, +see https://github.com/reframe-hpc/reframe/blob/develop/hpctestlib/sciapps/qespresso/benchmarks.py + +ReFrame terminology: + +"pipeline stages": +https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#pipeline-hooks + +"test parameter": a list of values, which will generate different test variants. +https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.builtins.parameter + +"test variant": a version of a test with a specific value for each test parameter +https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#test-variants + +"concrete test cases": all test combinations that will actually run: +- test variants +- valid system:partition+programming environment combinations +https://reframe-hpc.readthedocs.io/en/stable/tutorial_deps.html#listing-dependencies + +Tests can be filtered by name, tag, programming environment, system, partition, or maintainer, +see https://reframe-hpc.readthedocs.io/en/stable/manpage.html#test-filtering + +Hooks acting on all possible test combinations (before filtering) are called after the 'init' stage. +Hooks acting on concrete test cases (after filtering) are called after the 'setup' stage. + +See also https://reframe-hpc.readthedocs.io/en/stable/pipeline.html +""" + +import reframe as rfm +from hpctestlib.sciapps.qespresso.benchmarks import QEspressoPWCheck +from reframe.core.builtins import ( # added only to make the linter happy + parameter, run_after) + +from eessi.testsuite import hooks +from eessi.testsuite.constants import SCALES, TAGS +from eessi.testsuite.utils import find_modules, log + + +@rfm.simple_test +class EESSI_QuantumESPRESSO_PW(QEspressoPWCheck): + scale = parameter(SCALES.keys()) + valid_prog_environs = ['default'] + valid_systems = ['*'] + time_limit = '30m' + module_name = parameter(find_modules('QuantumESPRESSO')) + + @run_after('init') + def run_after_init(self): + """Hooks to run after the init phase""" + + # Filter on which scales are supported by the partitions defined in the ReFrame configuration + hooks.filter_supported_scales(self) + + # Make sure that GPU tests run in partitions that support running on a GPU, + # and that CPU-only tests run in partitions that support running CPU-only. + # Also support setting valid_systems on the cmd line. + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.nb_impl) + + # Support selecting modules on the cmd line. + hooks.set_modules(self) + + # Support selecting scales on the cmd line via tags. + hooks.set_tag_scale(self) + + @run_after('init') + def set_tag_ci(self): + """Set tag CI on smallest benchmark, so it can be selected on the cmd line via --tag CI""" + min_ecut = min(QEspressoPWCheck.ecut.values) + min_nbnd = min(QEspressoPWCheck.nbnd.values) + if self.ecut == min_ecut and self.nbnd == min_nbnd: + self.tags.add(TAGS['CI']) + log(f'tags set to {self.tags}') + + @run_after('setup') + def run_after_setup(self): + """Hooks to run after the setup phase""" + + # Calculate default requested resources based on the scale: + # 1 task per CPU for CPU-only tests, 1 task per GPU for GPU tests. + # Also support setting the resources on the cmd line. + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=self.nb_impl) + + @run_after('setup') + def set_omp_num_threads(self): + """ + Set number of OpenMP threads via OMP_NUM_THREADS. + Set default number of OpenMP threads equal to number of CPUs per task. + """ + + self.env_vars['OMP_NUM_THREADS'] = self.num_cpus_per_task + log(f'env_vars set to {self.env_vars}') From 36743e98fed4dbcd04d15ec37c4f3d822fb6c0cc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:50:56 +0100 Subject: [PATCH 013/119] Update README.md --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index cdec5fd4..5049bbf8 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,15 @@ origin git@github.com:EESSI/test-suite.git (fetch) origin git@github.com:EESSI/test-suite.git (push) ``` +#### Option 1: Creating a branch from the PR directly + +```bash +git fetch origin pull/ID/head:BRANCH_NAME +``` +where `ID` is the number of the pull request, and `BRANCH_NAME` is the name of the local branch (you can pick this yourself). + +#### Option 2: Creating a branch tracking the feature branch + You can add a fork to your local clone by adding a new remote. Pick a name for the remote that you find easy to recognize. E.g. to add the fork https://github.com/casparvl/test-suite and give it the (local) name `casparvl`, From 2e7bf95cc14ce3cf36552de53dbc6176c3625b2e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 27 Mar 2024 17:43:11 +0100 Subject: [PATCH 014/119] Initial commit for an EESSI PyTorch test that uses torchvision models --- .../tests/apps/PyTorch/PyTorch_torchvision.py | 153 +++++++++++++ .../src/pytorch_synthetic_benchmark.py | 213 ++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py create mode 100644 eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py new file mode 100644 index 00000000..91d0a708 --- /dev/null +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -0,0 +1,153 @@ +import reframe as rfm +import reframe.utility.sanity as sn + +from eessi.testsuite import hooks +from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, CPU_SOCKET, GPU +from eessi.testsuite.utils import find_modules, log + +class PyTorch_torchvision(rfm.RunOnlyRegressionTest): + nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large']) + ### SHOULD BE DETERMINED BY SCALE + #n_processes = parameter([1, 2, 4, 8, 16]) + scale = parameter(SCALES.keys()) + # Not sure how we would ensure the horovod module is _also_ loaded... + # parallel_strategy = parameter([None, 'horovod', 'ddp']) + parallel_strategy = parameter([None, 'ddp']) + compute_device = variable(str) + # module_name = parameter(find_modules('PyTorch-bundle')) + module_name = parameter(find_modules('torchvision')) + + descr = 'Benchmark that runs a selected torchvision model on synthetic data' + + executable = 'python' + + valid_prog_environs = ['default'] + valid_systems = ['*'] + + time_limit = '30m' + + @run_after('init') + def prepare_test(self): + + # Set nn_model as executable option + self.executable_opts = ['pytorch_synthetic_benchmark.py --model %s' % self.nn_model] + + # If not a GPU run, disable CUDA + if self.compute_device != DEVICE_TYPES[GPU]: + self.executable_opts += ['--no-cuda'] + + + + @run_after('init') + def apply_init_hooks(self): + # Filter on which scales are supported by the partitions defined in the ReFrame configuration + hooks.filter_supported_scales(self) + + # Make sure that GPU tests run in partitions that support running on a GPU, + # and that CPU-only tests run in partitions that support running CPU-only. + # Also support setting valid_systems on the cmd line. + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.compute_device) + + # Support selecting modules on the cmd line. + hooks.set_modules(self) + + # Support selecting scales on the cmd line via tags. + hooks.set_tag_scale(self) + + @run_after('init') + def set_tag_ci(self): + if self.nn_model == 'resnet50': + self.tags.add(TAGS['CI']) + + @run_after('setup') + def apply_setup_hooks(self): + if self.compute_device==DEVICE_TYPES[GPU]: + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[GPU]) + else: + # Hybrid code, so launch 1 rank per socket. + # Probably, launching 1 task per NUMA domain is even better, but the current hook doesn't support it + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[CPU_SOCKET]) + + # This is a hybrid test, binding is important for performance + hooks.set_compact_process_binding(self) + + @run_after('setup') + def set_ddp_env_vars(self): + # Set environment variables for PyTorch DDP + ### TODO: THIS WILL ONLY WORK WITH SLURM, WE SHOULD MAKE A SKIP_IF BASED ON THE SCHEDULER + if self.parallel_strategy == 'ddp': + self.prerun_cmds = [ + 'export MASTER_PORT=$(expr 10000 + $(echo -n $SLURM_JOBID | tail -c 4))', + 'export WORLD_SIZE=%s' % self.num_tasks, + 'echo "WORLD_SIZE="${WORLD_SIZE}', + 'master_addr=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)', + 'export MASTER_ADDR=${master_addr}', + 'echo "MASTER_ADDR"=${master_addr}', + ] + + + @run_after('setup') + def filter_invalid_parameter_combinations(self): + # We cannot detect this situation before the setup phase, because it requires self.num_tasks. + # Thus, the core count of the node needs to be known, which is only the case after the setup phase. + msg=f"Skipping test: parallel strategy is 'None', but requested process count is larger than one ({self.num_tasks})" + self.skip_if(self.num_tasks > 1 and self.parallel_strategy is None, msg) + msg=f"Skipping test: parallel strategy is {self.parallel_strategy}, but only one process is requested" + self.skip_if(self.num_tasks == 1 and not self.parallel_strategy is None, msg) + + @run_after('setup') + def pass_parallel_strategy(self): + # Set parallelization strategy when using more than one process + if self.num_tasks != 1: + self.executable_opts += ['--use-%s' % self.parallel_strategy] + + @run_after('setup') + def avoid_horovod_cpu_contention(self): + # Horovod had issues with CPU performance, see https://github.com/horovod/horovod/issues/2804 + # The root cause is Horovod having two threads with very high utilization, which interferes with + # the compute threads. It was fixed, but seems to be broken again in Horovod 0.28.1 + # The easiest workaround is to reduce the number of compute threads by 2 + if self.compute_device == DEVICE_TYPES[CPU] and self.parallel_strategy == 'horovod': + self.env_vars['OMP_NUM_THREADS'] = max(self.num_cpus_per_task-2, 2) # Never go below 2 compute threads + + @sanity_function + def assert_num_ranks(self): + '''Assert that the number of reported CPUs/GPUs used is correct''' + return sn.assert_found(r'Total img/sec on %s .PU\(s\):.*' % self.num_tasks, self.stdout) + + + @performance_function('img/sec') + def total_throughput(self): + '''Total training throughput, aggregated over all CPUs/GPUs''' + return sn.extractsingle(r'Total img/sec on [0-9]+ .PU\(s\):\s+(?P\S+)', self.stdout, 'perf', float) + + @performance_function('img/sec') + def througput_per_CPU(self): + '''Training througput per CPU''' + if self.compute_device == DEVICE_TYPES[CPU]: + return sn.extractsingle(r'Img/sec per CPU:\s+(?P\S+)', self.stdout, 'perf_per_cpu', float) + else: + return sn.extractsingle(r'Img/sec per GPU:\s+(?P\S+)', self.stdout, 'perf_per_gpu', float) + +@rfm.simple_test +class PyTorch_torchvision_CPU(PyTorch_torchvision): + compute_device = DEVICE_TYPES[CPU] + + +@rfm.simple_test +class PyTorch_torchvision_GPU(PyTorch_torchvision): + compute_device = DEVICE_TYPES[GPU] + precision = parameter(['default', 'mixed']) + + @run_after('init') + def prepare_gpu_test(self): + # Set precision + if self.precision == 'mixed': + self.executable_opts += ['--use-amp'] + + @run_after('init') + def skip_hvd_plus_amp(self): + '''Skip combination of horovod and AMP, it does not work see https://github.com/horovod/horovod/issues/1417''' + if self.parallel_strategy == 'horovod' and self.precision == 'mixed': + self.valid_systems = [] + diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py new file mode 100644 index 00000000..f3237e7d --- /dev/null +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -0,0 +1,213 @@ +from __future__ import print_function + +import argparse +import torch.backends.cudnn as cudnn +import torch.nn.functional as F +import torch.optim as optim +import torch.utils.data.distributed +from torchvision import models +import timeit +import numpy as np +import os + +# Benchmark settings +parser = argparse.ArgumentParser(description='PyTorch Synthetic Benchmark', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--fp16-allreduce', action='store_true', default=False, + help='use fp16 compression during allreduce') + +parser.add_argument('--model', type=str, default='resnet50', + help='model to benchmark') +parser.add_argument('--batch-size', type=int, default=32, + help='input batch size') + +parser.add_argument('--num-warmup-batches', type=int, default=10, + help='number of warm-up batches that don\'t count towards benchmark') +parser.add_argument('--num-batches-per-iter', type=int, default=10, + help='number of batches per benchmark iteration') +parser.add_argument('--num-iters', type=int, default=10, + help='number of benchmark iterations') + +parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + +parser.add_argument('--use-adasum', action='store_true', default=False, + help='use adasum algorithm to do reduction') +parser.add_argument('--use-horovod', action='store_true', default=False) +parser.add_argument('--use-ddp', action='store_true', default=False) + +parser.add_argument('--use-amp', action='store_true', default=False, + help='Use PyTorch Automatic Mixed Precision (AMP)') + +args = parser.parse_args() +args.cuda = not args.no_cuda and torch.cuda.is_available() + +if args.use_horovod and args.use_ddp: + print("You can't specify to use both Horovod and Pytorch DDP, exiting...") + exit(1) + +# Set a default rank and world size, also for when ddp and horovod are not used +rank = 0 +world_size=1 +if args.use_horovod: + import horovod.torch as hvd + hvd.init() + rank = hvd.local_rank() + world_size = hvd.size() + + if args.cuda: + # If launched with srun, you are in a CGROUP with only 1 GPU, so you don't need to set it. + # If launched with mpirun, you see ALL local GPUs on the node, and you need to set which one + # this rank should use. + visible_gpus = torch.cuda.device_count() + # Horovod: pin GPU to local rank. + if visible_gpus > 1: + torch.cuda.set_device(hvd.local_rank()) + + # Should only be uncommented for debugging + # In ReFrame tests, a print from each rank can mess up the output file, causing + # performance and sanity patterns to not be found + # print(f"hvd.local_rank: {rank}", flush=True) + + +if args.use_ddp: + import torch.distributed as dist + from torch.nn.parallel import DistributedDataParallel as DDP + from socket import gethostname + + def setup(rank, world_size): + # initialize the process group + if args.cuda: + dist.init_process_group("nccl", rank=rank, world_size=world_size) + else: + dist.init_process_group("gloo", rank=rank, world_size=world_size) + + def cleanup(): + # clean up the distributed environment + dist.destroy_process_group() + + world_size = int(os.environ["SLURM_NTASKS"]) + # If launched with mpirun, get rank from this + rank = int(os.environ.get("OMPI_COMM_WORLD_RANK", -1)) + if rank == -1: + # Else it's launched with srun, get rank from this + rank = int(os.environ["SLURM_PROCID"]) + + setup(rank, world_size) + # log(f"Group initialized? {dist.is_initialized()}", rank) + if rank == 0: print(f"Group initialized? {dist.is_initialized()}", flush=True) + + # If launched with srun, you are in a CGROUP with only 1 GPU, so you don't need to set it. + # If launched with mpirun, you see ALL local GPUs on the node, and you need to set which one + # this rank should use. + visible_gpus = torch.cuda.device_count() + if visible_gpus > 1: + local_rank = rank - visible_gpus * (rank // visible_gpus) + torch.cuda.set_device(local_rank) + print(f"host: {gethostname()}, rank: {rank}, local_rank: {local_rank}") + else: + print(f"host: {gethostname()}, rank: {rank}") + +# This relies on the 'rank' set in the if args.use_horovod or args.use_ddp sections +def log(s, nl=True): + if (args.use_horovod or args.use_ddp) and rank != 0: + return + print(s, end='\n' if nl else '', flush=True) + +log(f"World size: {world_size}") + +# Used to be needed, but now seems that different SLURM tasks run within their own cgroup +# Each cgroup only contains a single GPU, which has GPU ID 0. So no longer needed to set +# one of the ranks to GPU 0 and one to GPU 1 +#if args.cuda and args.use_horovod: +# # Horovod: pin GPU to local rank. +# torch.cuda.set_device(hvd.local_rank()) + +torch.set_num_threads(int(os.environ['OMP_NUM_THREADS'])) +torch.set_num_interop_threads(2) + +cudnn.benchmark = True + +# Set up standard model. +model = getattr(models, args.model)() + +# By default, Adasum doesn't need scaling up learning rate. +lr_scaler = hvd.size() if not args.use_adasum and args.use_horovod else 1 + +if args.cuda: + # Move model to GPU. + model.cuda() + # If using GPU Adasum allreduce, scale learning rate by local_size. + if args.use_horovod and args.use_adasum and hvd.nccl_built(): + lr_scaler = hvd.local_size() + +# If using DDP, wrap model +if args.use_ddp: + model = DDP(model) + +optimizer = optim.SGD(model.parameters(), lr=0.01 * lr_scaler) + +# Horovod: (optional) compression algorithm. +if args.use_horovod: + compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none + +# Horovod: wrap optimizer with DistributedOptimizer. +if args.use_horovod: + optimizer = hvd.DistributedOptimizer(optimizer, + named_parameters=model.named_parameters(), + compression=compression, + op=hvd.Adasum if args.use_adasum else hvd.Average) + + # Horovod: broadcast parameters & optimizer state. + hvd.broadcast_parameters(model.state_dict(), root_rank=0) + hvd.broadcast_optimizer_state(optimizer, root_rank=0) + +# Set up fixed fake data +data = torch.randn(args.batch_size, 3, 224, 224) +target = torch.LongTensor(args.batch_size).random_() % 1000 +if args.cuda: + data, target = data.cuda(), target.cuda() + +# Create GradScaler for automatic mixed precision +scaler = torch.cuda.amp.GradScaler(enabled=args.use_amp) + +# Set device_type for AMP +if args.cuda: + device_type="cuda" +else: + device_type="cpu" + +def benchmark_step(): + optimizer.zero_grad() + with torch.autocast(device_type=device_type, enabled=args.use_amp): + output = model(data) + loss = F.cross_entropy(output, target) + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + +log('Model: %s' % args.model) +log('Batch size: %d' % args.batch_size) +device = 'GPU' if args.cuda else 'CPU' +if args.use_horovod: + log('Number of %ss: %d' % (device, hvd.size())) + +# Warm-up +log('Running warmup...') +timeit.timeit(benchmark_step, number=args.num_warmup_batches) + +# Benchmark +log('Running benchmark...') +img_secs = [] +for x in range(args.num_iters): + time = timeit.timeit(benchmark_step, number=args.num_batches_per_iter) + img_sec = args.batch_size * args.num_batches_per_iter / time + log('Iter #%d: %.1f img/sec per %s' % (x, img_sec, device)) + img_secs.append(img_sec) + +# Results +img_sec_mean = np.mean(img_secs) +img_sec_conf = 1.96 * np.std(img_secs) +log('Img/sec per %s: %.1f +-%.1f' % (device, img_sec_mean, img_sec_conf)) +log('Total img/sec on %d %s(s): %.1f +-%.1f' % + (world_size, device, world_size * img_sec_mean, world_size * img_sec_conf)) From e089760e44eff8b88ede0d8bb38627392af3ac45 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 12:00:40 +0100 Subject: [PATCH 015/119] Add option to assign number of tasks and cpus per task based on the amount of numa nodes in a node --- eessi/testsuite/hooks.py | 62 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index c06ff572..883a0e4d 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -57,7 +57,7 @@ def _assign_default_num_gpus_per_node(test: rfm.RegressionTest): def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, num_per: int = 1): """ - Assign one task per compute unit (COMPUTE_UNIT[CPU], COMPUTE_UNIT[CPU_SOCKET] or COMPUTE_UNIT[GPU]). + Assign one task per compute unit. Automatically sets num_tasks, num_tasks_per_node, num_cpus_per_task, and num_gpus_per_node, based on the current scale and the current partition’s num_cpus, max_avail_gpus_per_node and num_nodes. For GPU tests, one task per GPU is set, and num_cpus_per_task is based on the ratio of CPU-cores/GPUs. @@ -109,6 +109,8 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n _assign_one_task_per_cpu(test) elif compute_unit == COMPUTE_UNIT[CPU_SOCKET]: _assign_one_task_per_cpu_socket(test) + elif compute_unit == COMPUTE_UNIT[NUMA_NODE]: + _assign_one_task_per_numa_node(test) elif compute_unit == COMPUTE_UNIT[NODE]: _assign_num_tasks_per_node(test, num_per) else: @@ -175,7 +177,7 @@ def _assign_one_task_per_cpu_socket(test: rfm.RegressionTest): test.num_tasks_per_node * test.num_cpus_per_task == test.default_num_cpus_per_node. Default resources requested: - - num_tasks_per_node = default_num_cpus_per_node + - num_tasks_per_node = default_num_cpus_per_node / num_cpus_per_socket - num_cpus_per_task = default_num_cpus_per_node / num_tasks_per_node """ # neither num_tasks_per_node nor num_cpus_per_task are set @@ -206,6 +208,62 @@ def _assign_one_task_per_cpu_socket(test: rfm.RegressionTest): log(f'num_tasks set to {test.num_tasks}') +def _assign_one_task_per_numa_node(test: rfm.RegressionTest): + """ + Determines the number of tasks per node by dividing the default_num_cpus_per_node by + the number of cpus available per numa node, and rounding up. The result is that for full-node jobs the default + will spawn one task per numa node, with a number of cpus per task equal to the number of cpus per numa node. + Other examples: + - half a node (i.e. node_part=2) on a system with 4 numa nodes would result in 2 tasks per node, + with number of cpus per task equal to the number of cpus per numa node. + - a quarter node (i.e. node_part=4) on a system with 2 numa nodes would result in 1 task per node, + with number of cpus equal to half a numa node. + - 2 cores (i.e. default_num_cpus_per_node=2) on a system with 4 cores per numa node would result in + 1 task per node, with 2 cpus per task + - 8 cores (i.e. default_num_cpus_per_node=4) on a system with 4 cores per numa node would result in + 2 tasks per node, with 4 cpus per task + + This default is set unless the test is run with: + --setvar num_tasks_per_node= and/or + --setvar num_cpus_per_task=. + In those cases, those take precedence, and the remaining variable (num_cpus_per task or + num_tasks_per_node respectively) is calculated based on the equality + test.num_tasks_per_node * test.num_cpus_per_task == test.default_num_cpus_per_node. + + Default resources requested: + - num_tasks_per_node = default_num_cpus_per_node / num_cores_per_numa_node + - num_cpus_per_task = default_num_cpus_per_node / num_tasks_per_node + """ + # neither num_tasks_per_node nor num_cpus_per_task are set + if not test.num_tasks_per_node and not test.num_cpus_per_task: + # Not needed, if num_cores_per_numa_node is really defined by reframe... https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.systems.ProcessorInfo + # check_proc_attribute_defined(test, 'num_cpus') + check_proc_attribute_defined(test, 'num_cores_per_numa_node') + # num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets + test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / num_cores_per_numa_node) + test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) + + # num_tasks_per_node is not set, but num_cpus_per_task is + elif not test.num_tasks_per_node: + # check_proc_attribute_defined(test, 'num_cpus') + check_proc_attribute_defined(test, 'num_cores_per_numa_node') + # num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets # Unused? + test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.num_cpus_per_task) + + # num_cpus_per_task is not set, but num_tasks_per_node is + elif not test.num_cpus_per_task: + test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) + + else: + pass # both num_tasks_per_node and num_cpus_per_node are already set + + test.num_tasks = test.num_nodes * test.num_tasks_per_node + log(f'Number of tasks per node set to: {test.num_tasks_per_node}') + log(f'Number of cpus per task set to {test.num_cpus_per_task}') + log(f'num_tasks set to {test.num_tasks}') + + + def _assign_one_task_per_cpu(test: rfm.RegressionTest): """ Sets num_tasks_per_node and num_cpus_per_task such that it will run one task per core, From a675fc976437cb533c0f1ff5d80903b16c31925c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 12:09:14 +0100 Subject: [PATCH 016/119] We don't need to first calculate cpus_per_socket, it is available directly from the refraem.core.systems.ProcessorInfo object. See https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.systems.ProcessorInfo --- eessi/testsuite/hooks.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 883a0e4d..5956ac61 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -182,17 +182,12 @@ def _assign_one_task_per_cpu_socket(test: rfm.RegressionTest): """ # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: - check_proc_attribute_defined(test, 'num_cpus') - check_proc_attribute_defined(test, 'num_sockets') - num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets - test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / num_cpus_per_socket) + check_proc_attribute_defined(test, 'num_cores_per_socket') + test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_socket) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is elif not test.num_tasks_per_node: - check_proc_attribute_defined(test, 'num_cpus') - check_proc_attribute_defined(test, 'num_sockets') - num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.num_cpus_per_task) # num_cpus_per_task is not set, but num_tasks_per_node is @@ -236,18 +231,12 @@ def _assign_one_task_per_numa_node(test: rfm.RegressionTest): """ # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: - # Not needed, if num_cores_per_numa_node is really defined by reframe... https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.systems.ProcessorInfo - # check_proc_attribute_defined(test, 'num_cpus') check_proc_attribute_defined(test, 'num_cores_per_numa_node') - # num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / num_cores_per_numa_node) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is elif not test.num_tasks_per_node: - # check_proc_attribute_defined(test, 'num_cpus') - check_proc_attribute_defined(test, 'num_cores_per_numa_node') - # num_cpus_per_socket = test.current_partition.processor.num_cpus / test.current_partition.processor.num_sockets # Unused? test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.num_cpus_per_task) # num_cpus_per_task is not set, but num_tasks_per_node is From a6e53bc0124b7163dfede9d430c4d62046e4670b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 12:09:37 +0100 Subject: [PATCH 017/119] Define constant for numa node as a compute unit --- eessi/testsuite/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi/testsuite/constants.py b/eessi/testsuite/constants.py index 9b7d6ac3..5254f710 100644 --- a/eessi/testsuite/constants.py +++ b/eessi/testsuite/constants.py @@ -6,6 +6,7 @@ CI = 'CI' CPU = 'CPU' CPU_SOCKET = 'CPU_SOCKET' +NUMA_NODE = 'NUMA_NODE' GPU = 'GPU' GPU_VENDOR = 'GPU_VENDOR' INTEL = 'INTEL' @@ -21,6 +22,7 @@ COMPUTE_UNIT = { CPU: 'cpu', CPU_SOCKET: 'cpu_socket', + NUMA_NODE: 'numa_node', GPU: 'gpu', NODE: 'node', } From 8bb9bbd8bd631b387be3566b7b902138d495bc10 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 12:26:36 +0100 Subject: [PATCH 018/119] Fix num_cores_per_numa_node, use it from the current_partition info --- eessi/testsuite/hooks.py | 2 +- .../tests/apps/PyTorch/src/python_get_free_socket.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 5956ac61..b9a3dc93 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -232,7 +232,7 @@ def _assign_one_task_per_numa_node(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cores_per_numa_node') - test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / num_cores_per_numa_node) + test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_numa_node) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is diff --git a/eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py b/eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py new file mode 100644 index 00000000..a2981304 --- /dev/null +++ b/eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py @@ -0,0 +1,8 @@ +# Based on https://unix.stackexchange.com/a/132524 +import socket + +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.bind(('', 0)) +addr = s.getsockname() +print(addr[1]) +s.close() From a6bf34d3ad79a2f4734cdede08ac9cca12562944 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 29 Mar 2024 12:28:19 +0100 Subject: [PATCH 019/119] Work around the issue of not being able to export varialbes in a launcher-agnostic way, by simply passing them to the python script as argument, and having that set it to the environment. Print clear error if SLURM or openMPIs mpirun are not used - we still rely on these to get the local rank, there is no other way --- .../tests/apps/PyTorch/PyTorch_torchvision.py | 19 ++++------- .../src/pytorch_synthetic_benchmark.py | 34 ++++++++++++++----- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 91d0a708..6367c63e 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -2,7 +2,7 @@ import reframe.utility.sanity as sn from eessi.testsuite import hooks -from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, CPU_SOCKET, GPU +from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM from eessi.testsuite.utils import find_modules, log class PyTorch_torchvision(rfm.RunOnlyRegressionTest): @@ -66,7 +66,7 @@ def apply_setup_hooks(self): else: # Hybrid code, so launch 1 rank per socket. # Probably, launching 1 task per NUMA domain is even better, but the current hook doesn't support it - hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[CPU_SOCKET]) + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[NUMA_NODE]) # This is a hybrid test, binding is important for performance hooks.set_compact_process_binding(self) @@ -76,15 +76,10 @@ def set_ddp_env_vars(self): # Set environment variables for PyTorch DDP ### TODO: THIS WILL ONLY WORK WITH SLURM, WE SHOULD MAKE A SKIP_IF BASED ON THE SCHEDULER if self.parallel_strategy == 'ddp': - self.prerun_cmds = [ - 'export MASTER_PORT=$(expr 10000 + $(echo -n $SLURM_JOBID | tail -c 4))', - 'export WORLD_SIZE=%s' % self.num_tasks, - 'echo "WORLD_SIZE="${WORLD_SIZE}', - 'master_addr=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)', - 'export MASTER_ADDR=${master_addr}', - 'echo "MASTER_ADDR"=${master_addr}', - ] - + # Set additional options required by DDP + self.executable_opts += ["--master-port $(python python_get_free_socket.py)"] + self.executable_opts += ["--master-address $(hostname --fqdn)"] + self.executable_opts += ["--world-size %s" % self.num_tasks] @run_after('setup') def filter_invalid_parameter_combinations(self): @@ -149,5 +144,5 @@ def prepare_gpu_test(self): def skip_hvd_plus_amp(self): '''Skip combination of horovod and AMP, it does not work see https://github.com/horovod/horovod/issues/1417''' if self.parallel_strategy == 'horovod' and self.precision == 'mixed': - self.valid_systems = [] + self.valid_systems = [INVALID_SYSTEM] diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py index f3237e7d..4c0db3be 100644 --- a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -1,14 +1,15 @@ -from __future__ import print_function - import argparse +import timeit +import os +import random + +import numpy as np + import torch.backends.cudnn as cudnn import torch.nn.functional as F import torch.optim as optim import torch.utils.data.distributed from torchvision import models -import timeit -import numpy as np -import os # Benchmark settings parser = argparse.ArgumentParser(description='PyTorch Synthetic Benchmark', @@ -38,6 +39,12 @@ parser.add_argument('--use-amp', action='store_true', default=False, help='Use PyTorch Automatic Mixed Precision (AMP)') +parser.add_argument('--world-size', type=int, default=1, + help='Define the world size for ddp') +parser.add_argument('--master-port', type=int, default=False, + help='Define a master port for ddp') +parser.add_argument('--master-address', type=str, default='localhost', + help='Define a master address for ddp') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() @@ -46,9 +53,15 @@ print("You can't specify to use both Horovod and Pytorch DDP, exiting...") exit(1) +# Set MASTER_ADDR and MASTER_PORT environment variables +# By doing it as part of this python script, we don't need to have the launchers export them +# This saves us from having to find a launcher-agnostic way of exporting variables +os.environ['MASTER_ADDR'] = args.master_address +os.environ['MASTER_PORT'] = '%s' % args.master_port + # Set a default rank and world size, also for when ddp and horovod are not used rank = 0 -world_size=1 +world_size = args.world_size if args.use_horovod: import horovod.torch as hvd hvd.init() @@ -86,12 +99,17 @@ def cleanup(): # clean up the distributed environment dist.destroy_process_group() - world_size = int(os.environ["SLURM_NTASKS"]) + # world_size = int(os.environ["SLURM_NTASKS"]) ## No longer needed now we pass it as argument? # If launched with mpirun, get rank from this rank = int(os.environ.get("OMPI_COMM_WORLD_RANK", -1)) if rank == -1: # Else it's launched with srun, get rank from this - rank = int(os.environ["SLURM_PROCID"]) + rank = int(os.environ.get("SLURM_PROCID", -1)) + if rank == -1: + err_msg = "ERROR: cannot determine local rank. This test currently only supports OpenMPI" + err_msg += " and srun as launchers. If you've configured a different launcher for your system" + err_msg += " this test will need to be extended with a method to get it's local rank for that launcher." + print(err_msg) setup(rank, world_size) # log(f"Group initialized? {dist.is_initialized()}", rank) From 67e61e44ead58f449898a22102681e8e69c2062f Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 3 Apr 2024 17:24:44 +0200 Subject: [PATCH 020/119] Implemented use of `DEVICE_TYPES` --- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index a7bb6dd4..b5b3db25 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -34,7 +34,8 @@ parameter, run_after) from eessi.testsuite import hooks -from eessi.testsuite.constants import SCALES, TAGS +from eessi.testsuite.constants import (COMPUTE_UNIT, CPU, DEVICE_TYPES, GPU, + SCALES, TAGS) from eessi.testsuite.utils import find_modules, log @@ -45,6 +46,9 @@ class EESSI_QuantumESPRESSO_PW(QEspressoPWCheck): valid_systems = ['*'] time_limit = '30m' module_name = parameter(find_modules('QuantumESPRESSO')) + # For now, QE is being build for CPU targets only + # compute_device = parameter([DEVICE_TYPES[CPU], DEVICE_TYPES[GPU]]) + compute_device = parameter([DEVICE_TYPES[CPU], ]) @run_after('init') def run_after_init(self): @@ -56,7 +60,7 @@ def run_after_init(self): # Make sure that GPU tests run in partitions that support running on a GPU, # and that CPU-only tests run in partitions that support running CPU-only. # Also support setting valid_systems on the cmd line. - hooks.filter_valid_systems_by_device_type(self, required_device_type=self.nb_impl) + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.compute_device) # Support selecting modules on the cmd line. hooks.set_modules(self) @@ -80,7 +84,10 @@ def run_after_setup(self): # Calculate default requested resources based on the scale: # 1 task per CPU for CPU-only tests, 1 task per GPU for GPU tests. # Also support setting the resources on the cmd line. - hooks.assign_tasks_per_compute_unit(test=self, compute_unit=self.nb_impl) + if self.compute_device == DEVICE_TYPES[GPU]: + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[GPU]) + else: + hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[CPU]) @run_after('setup') def set_omp_num_threads(self): From fce2a4518dbe43591abc5ec6d3992ffd3dac20bb Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 9 Apr 2024 10:50:47 +0200 Subject: [PATCH 021/119] Change order of imports so that initialization only happens after required environment variables have been set. --- .../src/pytorch_synthetic_benchmark.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py index 4c0db3be..78ba43df 100644 --- a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -11,6 +11,7 @@ import torch.utils.data.distributed from torchvision import models + # Benchmark settings parser = argparse.ArgumentParser(description='PyTorch Synthetic Benchmark', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -84,20 +85,9 @@ if args.use_ddp: + from socket import gethostname import torch.distributed as dist from torch.nn.parallel import DistributedDataParallel as DDP - from socket import gethostname - - def setup(rank, world_size): - # initialize the process group - if args.cuda: - dist.init_process_group("nccl", rank=rank, world_size=world_size) - else: - dist.init_process_group("gloo", rank=rank, world_size=world_size) - - def cleanup(): - # clean up the distributed environment - dist.destroy_process_group() # world_size = int(os.environ["SLURM_NTASKS"]) ## No longer needed now we pass it as argument? # If launched with mpirun, get rank from this @@ -110,22 +100,46 @@ def cleanup(): err_msg += " and srun as launchers. If you've configured a different launcher for your system" err_msg += " this test will need to be extended with a method to get it's local rank for that launcher." print(err_msg) - - setup(rank, world_size) - # log(f"Group initialized? {dist.is_initialized()}", rank) - if rank == 0: print(f"Group initialized? {dist.is_initialized()}", flush=True) # If launched with srun, you are in a CGROUP with only 1 GPU, so you don't need to set it. # If launched with mpirun, you see ALL local GPUs on the node, and you need to set which one # this rank should use. visible_gpus = torch.cuda.device_count() if visible_gpus > 1: + print("Listing visible devices") + for i in range(torch.cuda.device_count()): + print(f"Device {i}: {torch.cuda.device(i)}") local_rank = rank - visible_gpus * (rank // visible_gpus) torch.cuda.set_device(local_rank) + print("Listing visible devices after setting one") + for i in range(torch.cuda.device_count()): + print(f"Device {i}: {torch.cuda.device(i)}") + # We should also set CUDA_VISIBLE_DEVICES, which gets respected by NCCL + os.environ['CUDA_VISIBLE_DEVICES'] = '%s' % local_rank print(f"host: {gethostname()}, rank: {rank}, local_rank: {local_rank}") else: print(f"host: {gethostname()}, rank: {rank}") + + def setup(rank, world_size): + + # initialize the process group + if args.cuda: + dist.init_process_group("nccl", rank=rank, world_size=world_size) + else: + dist.init_process_group("gloo", rank=rank, world_size=world_size) + + def cleanup(): + # clean up the distributed environment + dist.destroy_process_group() + + setup(rank, world_size) + # log(f"Group initialized? {dist.is_initialized()}", rank) + if rank == 0: print(f"Group initialized? {dist.is_initialized()}", flush=True) + + + + # This relies on the 'rank' set in the if args.use_horovod or args.use_ddp sections def log(s, nl=True): if (args.use_horovod or args.use_ddp) and rank != 0: From 74dd9cfe76e895663d819e1680a437e3e0059dcb Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 18 Apr 2024 11:24:20 +0200 Subject: [PATCH 022/119] Prevent running on forks --- .github/workflows/scorecards.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index dc18fd58..39cbade2 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -22,6 +22,7 @@ permissions: read-all jobs: analysis: + if: github.repository == 'EESSI/test-suite' # Prevent running on forks name: Scorecards analysis runs-on: ubuntu-latest permissions: From b868cc14b4eb2be2ee721cec531baef4183eeec0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 3 May 2024 15:45:36 +0200 Subject: [PATCH 023/119] Add comment on explicit assumption for computing the local rank --- .../tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py index 78ba43df..c349b089 100644 --- a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -109,6 +109,10 @@ print("Listing visible devices") for i in range(torch.cuda.device_count()): print(f"Device {i}: {torch.cuda.device(i)}") + # This assumes compact mapping of ranks to available hardware + # e.g. rank 0-x to node 1, rank x-y to node 2, etc + # Assuming the set_compact_process_binding hook from the EESSI testsuite is called, + # this condition should be satisfied local_rank = rank - visible_gpus * (rank // visible_gpus) torch.cuda.set_device(local_rank) print("Listing visible devices after setting one") From 6ecad3a4fcb1b3d402d95b69015776ff1def8492 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 3 May 2024 16:33:54 +0200 Subject: [PATCH 024/119] Currently, the mapping was set by node. That is NOT the same thing as compact process binding, since for e.g. 2 nodes 4 tasks it will lead to task 0 on node 0, task 1 on node 1, task 2 on node 0, task 3 on node 1. Compact would have been task 0 and 1 on node 0, and task 2 and 3 on node 1. Thus, this was a bug. Mapping by slot (combined with specifying PE=n) IS correct. The PE=n will set cpus-per-rank to n (see https://www.open-mpi.org/doc/current/man1/mpirun.1.php), while mapping by slot will mean that each rank is mapped to a consecutive slot. The only scenario in which mapping by slot will result in non-compact mapping is if oversubscription is enabled - then the oversubscribed ranks will be assigned round robin to the slots. --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index c06ff572..0b73cd58 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -436,7 +436,7 @@ def set_compact_process_binding(test: rfm.RegressionTest): # Other launchers may or may not do the correct binding test.env_vars['I_MPI_PIN_CELL'] = 'core' # Don't bind to hyperthreads, only to physcial cores test.env_vars['I_MPI_PIN_DOMAIN'] = '%s:compact' % physical_cpus_per_task - test.env_vars['OMPI_MCA_rmaps_base_mapping_policy'] = 'node:PE=%s' % physical_cpus_per_task + test.env_vars['OMPI_MCA_rmaps_base_mapping_policy'] = 'slot:PE=%s' % physical_cpus_per_task # Default binding for SLURM. Only effective if the task/affinity plugin is enabled # and when number of tasks times cpus per task equals either socket, core or thread count test.env_vars['SLURM_CPU_BIND'] = 'verbose' From 4d8e10fd4a3b130339e5e2374a4162114e29c637 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 3 May 2024 18:30:59 +0200 Subject: [PATCH 025/119] Set process binding for GROMACS. It is single core, but if launched with mpirun it is currently free to migrate between cores within a numa domain. On Snellius I've seen some strange issues with occassionally very slow performance (10x slower than normal), potentially due to the OS thread schedulling being silly. Process binding leads to better _and_ more reproducible results --- eessi/testsuite/tests/apps/gromacs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eessi/testsuite/tests/apps/gromacs.py b/eessi/testsuite/tests/apps/gromacs.py index a3d9e625..c10da7c6 100644 --- a/eessi/testsuite/tests/apps/gromacs.py +++ b/eessi/testsuite/tests/apps/gromacs.py @@ -113,3 +113,11 @@ def set_omp_num_threads(self): self.env_vars['OMP_NUM_THREADS'] = omp_num_threads log(f'env_vars set to {self.env_vars}') + + @run_after('setup') + def set_binding_policy(self): + """ + Default process binding may depend on the launcher used. We've seen some variable performance. + Better set it explicitely to make sure process migration cannot cause such variations. + """ + hooks.set_compact_process_binding(self) From 241eabbce760570bdc01ad76f1ea60fe249807da Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 3 May 2024 18:46:18 +0200 Subject: [PATCH 026/119] Explicitely add num_cpus_per_core=1. I don't actually know if hyperthreading is enabled in the github CI environment, but it doesn't really matter for the dry-runs anyway. --- config/github_actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/github_actions.py b/config/github_actions.py index 5328f6f3..9371e376 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -18,7 +18,10 @@ 'launcher': 'local', 'environs': ['default'], 'features': [FEATURES[CPU]] + list(SCALES.keys()), - 'processor': {'num_cpus': 2}, + 'processor': { + 'num_cpus': 2, + 'num_cpus_per_core': 1, + }, 'resources': [ { 'name': 'memory', From 2ff34841b0bd4978ab2af2a0128c5784244cba7a Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 5 May 2024 09:58:46 +0200 Subject: [PATCH 027/119] rename 1_cpn_2_nodes and 1_cpn_4_nodes to ensure unique tag matching --- eessi/testsuite/constants.py | 4 ++-- eessi/testsuite/tests/apps/osu.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/constants.py b/eessi/testsuite/constants.py index 9b7d6ac3..dbf97f54 100644 --- a/eessi/testsuite/constants.py +++ b/eessi/testsuite/constants.py @@ -51,8 +51,8 @@ '1_core': {'num_nodes': 1, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, '2_cores': {'num_nodes': 1, 'num_cpus_per_node': 2, 'num_gpus_per_node': 1}, '4_cores': {'num_nodes': 1, 'num_cpus_per_node': 4, 'num_gpus_per_node': 1}, - '1_cpn_2_nodes': {'num_nodes': 2, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, - '1_cpn_4_nodes': {'num_nodes': 4, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, + '1cpn_2nodes': {'num_nodes': 2, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, + '1cpn_4nodes': {'num_nodes': 4, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, '1_8_node': {'num_nodes': 1, 'node_part': 8}, # 1/8 node '1_4_node': {'num_nodes': 1, 'node_part': 4}, # 1/4 node '1_2_node': {'num_nodes': 1, 'node_part': 2}, # 1/2 node diff --git a/eessi/testsuite/tests/apps/osu.py b/eessi/testsuite/tests/apps/osu.py index 3a6f9dbe..2fab94d7 100644 --- a/eessi/testsuite/tests/apps/osu.py +++ b/eessi/testsuite/tests/apps/osu.py @@ -114,7 +114,7 @@ def set_tag_ci(self): @run_after('init') def set_mem(self): """ Setting an extra job option of memory. This test has only 4 possibilities: 1_node, 2_nodes, 2_cores and - 1_cpn_2_nodes. This is implemented for all cases including full node cases. The requested memory may seem large + 1cpn_2nodes. This is implemented for all cases including full node cases. The requested memory may seem large and the test requires at least 4.5 GB per core at the minimum for the full test when run with validation (-c option for osu_bw or osu_latency). We run till message size 8 (-m 8) which significantly reduces memory requirement.""" From 458379e0e1f5178944d2ff76b29d63d16ab6113c Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 5 May 2024 11:57:24 +0200 Subject: [PATCH 028/119] add comments --- eessi/testsuite/constants.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi/testsuite/constants.py b/eessi/testsuite/constants.py index dbf97f54..19ad4a3d 100644 --- a/eessi/testsuite/constants.py +++ b/eessi/testsuite/constants.py @@ -51,7 +51,9 @@ '1_core': {'num_nodes': 1, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, '2_cores': {'num_nodes': 1, 'num_cpus_per_node': 2, 'num_gpus_per_node': 1}, '4_cores': {'num_nodes': 1, 'num_cpus_per_node': 4, 'num_gpus_per_node': 1}, + # renamed after v0.2.0 from 1_cpn_2_nodes to make more unique '1cpn_2nodes': {'num_nodes': 2, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, + # renamed after v0.2.0 from 1_cpn_4_nodes to make more unique '1cpn_4nodes': {'num_nodes': 4, 'num_cpus_per_node': 1, 'num_gpus_per_node': 1}, '1_8_node': {'num_nodes': 1, 'node_part': 8}, # 1/8 node '1_4_node': {'num_nodes': 1, 'node_part': 4}, # 1/4 node From fed890617e3ed556f23b72b80e0e11246a88d4f0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 11:04:35 +0200 Subject: [PATCH 029/119] Use EESSI prefix to name test --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 6367c63e..ef5a854c 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -5,7 +5,7 @@ from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM from eessi.testsuite.utils import find_modules, log -class PyTorch_torchvision(rfm.RunOnlyRegressionTest): +class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest): nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large']) ### SHOULD BE DETERMINED BY SCALE #n_processes = parameter([1, 2, 4, 8, 16]) From 357f649296361339a127025cfbd0efc071d8058e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 11:08:50 +0200 Subject: [PATCH 030/119] Child classes should also be renamed and inherit from renamed class --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index ef5a854c..f7e4286b 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -125,12 +125,12 @@ def througput_per_CPU(self): return sn.extractsingle(r'Img/sec per GPU:\s+(?P\S+)', self.stdout, 'perf_per_gpu', float) @rfm.simple_test -class PyTorch_torchvision_CPU(PyTorch_torchvision): +class EESSI_PyTorch_torchvision_CPU(EESSI_PyTorch_torchvision): compute_device = DEVICE_TYPES[CPU] @rfm.simple_test -class PyTorch_torchvision_GPU(PyTorch_torchvision): +class EESSI_PyTorch_torchvision_GPU(EESSI_PyTorch_torchvision): compute_device = DEVICE_TYPES[GPU] precision = parameter(['default', 'mixed']) From 6b1e36ae63bdc49c3812ae5bac81ac063547da97 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 14:44:56 +0200 Subject: [PATCH 031/119] Remove stray blank line --- eessi/testsuite/hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 1043a425..73306286 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -252,7 +252,6 @@ def _assign_one_task_per_numa_node(test: rfm.RegressionTest): log(f'num_tasks set to {test.num_tasks}') - def _assign_one_task_per_cpu(test: rfm.RegressionTest): """ Sets num_tasks_per_node and num_cpus_per_task such that it will run one task per core, From 2d3314174ab51d1867d1815edfcd2cc2cd0d9822 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 15:29:58 +0200 Subject: [PATCH 032/119] Rephrased comment, some changes to make the linter happy --- .../tests/apps/PyTorch/PyTorch_torchvision.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index f7e4286b..8011cd0d 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -1,21 +1,19 @@ import reframe as rfm import reframe.utility.sanity as sn +from reframe.core.builtins import parameter, run_after # added only to make the linter happy from eessi.testsuite import hooks from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM -from eessi.testsuite.utils import find_modules, log +from eessi.testsuite.utils import find_modules + class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest): nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large']) - ### SHOULD BE DETERMINED BY SCALE - #n_processes = parameter([1, 2, 4, 8, 16]) scale = parameter(SCALES.keys()) - # Not sure how we would ensure the horovod module is _also_ loaded... - # parallel_strategy = parameter([None, 'horovod', 'ddp']) parallel_strategy = parameter([None, 'ddp']) compute_device = variable(str) - # module_name = parameter(find_modules('PyTorch-bundle')) - module_name = parameter(find_modules('torchvision')) + # Both torchvision and PyTorch-bundle modules have everything needed to run this test + module_name = parameter(find_modules('torchvision') + find_modules('PyTorch-bundle')) descr = 'Benchmark that runs a selected torchvision model on synthetic data' @@ -36,8 +34,6 @@ def prepare_test(self): if self.compute_device != DEVICE_TYPES[GPU]: self.executable_opts += ['--no-cuda'] - - @run_after('init') def apply_init_hooks(self): # Filter on which scales are supported by the partitions defined in the ReFrame configuration @@ -74,7 +70,6 @@ def apply_setup_hooks(self): @run_after('setup') def set_ddp_env_vars(self): # Set environment variables for PyTorch DDP - ### TODO: THIS WILL ONLY WORK WITH SLURM, WE SHOULD MAKE A SKIP_IF BASED ON THE SCHEDULER if self.parallel_strategy == 'ddp': # Set additional options required by DDP self.executable_opts += ["--master-port $(python python_get_free_socket.py)"] @@ -109,7 +104,6 @@ def avoid_horovod_cpu_contention(self): def assert_num_ranks(self): '''Assert that the number of reported CPUs/GPUs used is correct''' return sn.assert_found(r'Total img/sec on %s .PU\(s\):.*' % self.num_tasks, self.stdout) - @performance_function('img/sec') def total_throughput(self): @@ -124,6 +118,7 @@ def througput_per_CPU(self): else: return sn.extractsingle(r'Img/sec per GPU:\s+(?P\S+)', self.stdout, 'perf_per_gpu', float) + @rfm.simple_test class EESSI_PyTorch_torchvision_CPU(EESSI_PyTorch_torchvision): compute_device = DEVICE_TYPES[CPU] From 4cb7b360ceeed5de4859328f4850a63b9724ebe3 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 16:25:00 +0200 Subject: [PATCH 033/119] Fix some more linter issues --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 8011cd0d..b2aa36bf 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -1,6 +1,6 @@ import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.builtins import parameter, run_after # added only to make the linter happy +from reframe.core.builtins import parameter, variable, run_after, sanity_function, performance_function # added only to make the linter happy from eessi.testsuite import hooks from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM From 2f0bea270f5c520b9e6ba92f47d767b8f7842002 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 16:39:27 +0200 Subject: [PATCH 034/119] Fix some more linter issues --- .../tests/apps/PyTorch/PyTorch_torchvision.py | 13 +++++++------ .../apps/PyTorch/src/pytorch_synthetic_benchmark.py | 13 ++++--------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index b2aa36bf..44cdd7f4 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -53,7 +53,7 @@ def apply_init_hooks(self): @run_after('init') def set_tag_ci(self): if self.nn_model == 'resnet50': - self.tags.add(TAGS['CI']) + self.tags.add(TAGS['CI']) @run_after('setup') def apply_setup_hooks(self): @@ -80,10 +80,12 @@ def set_ddp_env_vars(self): def filter_invalid_parameter_combinations(self): # We cannot detect this situation before the setup phase, because it requires self.num_tasks. # Thus, the core count of the node needs to be known, which is only the case after the setup phase. - msg=f"Skipping test: parallel strategy is 'None', but requested process count is larger than one ({self.num_tasks})" + msg = f"Skipping test: parallel strategy is 'None'," + msg += f" but requested process count is larger than one ({self.num_tasks})." self.skip_if(self.num_tasks > 1 and self.parallel_strategy is None, msg) - msg=f"Skipping test: parallel strategy is {self.parallel_strategy}, but only one process is requested" - self.skip_if(self.num_tasks == 1 and not self.parallel_strategy is None, msg) + msg = f"Skipping test: parallel strategy is {self.parallel_strategy}," + msg += f" but only one process is requested." + self.skip_if(self.num_tasks == 1 and self.parallel_strategy is not None, msg) @run_after('setup') def pass_parallel_strategy(self): @@ -98,7 +100,7 @@ def avoid_horovod_cpu_contention(self): # the compute threads. It was fixed, but seems to be broken again in Horovod 0.28.1 # The easiest workaround is to reduce the number of compute threads by 2 if self.compute_device == DEVICE_TYPES[CPU] and self.parallel_strategy == 'horovod': - self.env_vars['OMP_NUM_THREADS'] = max(self.num_cpus_per_task-2, 2) # Never go below 2 compute threads + self.env_vars['OMP_NUM_THREADS'] = max(self.num_cpus_per_task - 2, 2) # Never go below 2 compute threads @sanity_function def assert_num_ranks(self): @@ -140,4 +142,3 @@ def skip_hvd_plus_amp(self): '''Skip combination of horovod and AMP, it does not work see https://github.com/horovod/horovod/issues/1417''' if self.parallel_strategy == 'horovod' and self.precision == 'mixed': self.valid_systems = [INVALID_SYSTEM] - diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py index c349b089..b414d549 100644 --- a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -1,7 +1,6 @@ import argparse import timeit import os -import random import numpy as np @@ -124,7 +123,6 @@ else: print(f"host: {gethostname()}, rank: {rank}") - def setup(rank, world_size): # initialize the process group @@ -132,17 +130,14 @@ def setup(rank, world_size): dist.init_process_group("nccl", rank=rank, world_size=world_size) else: dist.init_process_group("gloo", rank=rank, world_size=world_size) - + def cleanup(): # clean up the distributed environment dist.destroy_process_group() - - setup(rank, world_size) - # log(f"Group initialized? {dist.is_initialized()}", rank) - if rank == 0: print(f"Group initialized? {dist.is_initialized()}", flush=True) - - + setup(rank, world_size) + if rank == 0: + print(f"Group initialized? {dist.is_initialized()}", flush=True) # This relies on the 'rank' set in the if args.use_horovod or args.use_ddp sections def log(s, nl=True): From fc067b218a02fe3fdbf700fdec16287dfec882bd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 16:51:23 +0200 Subject: [PATCH 035/119] Fix linter issues --- eessi/testsuite/hooks.py | 8 ++++++-- .../PyTorch/src/pytorch_synthetic_benchmark.py | 15 ++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 73306286..4bcdf6e6 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -183,7 +183,9 @@ def _assign_one_task_per_cpu_socket(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cores_per_socket') - test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_socket) + test.num_tasks_per_node = math.ceil( + test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_socket + ) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is @@ -232,7 +234,9 @@ def _assign_one_task_per_numa_node(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cores_per_numa_node') - test.num_tasks_per_node = math.ceil(test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_numa_node) + test.num_tasks_per_node = math.ceil( + test.default_num_cpus_per_node / test.current_partition.processor.num_cores_per_numa_node + ) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is diff --git a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py index b414d549..373790b5 100644 --- a/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py +++ b/eessi/testsuite/tests/apps/PyTorch/src/pytorch_synthetic_benchmark.py @@ -139,20 +139,15 @@ def cleanup(): if rank == 0: print(f"Group initialized? {dist.is_initialized()}", flush=True) + # This relies on the 'rank' set in the if args.use_horovod or args.use_ddp sections def log(s, nl=True): if (args.use_horovod or args.use_ddp) and rank != 0: return print(s, end='\n' if nl else '', flush=True) -log(f"World size: {world_size}") -# Used to be needed, but now seems that different SLURM tasks run within their own cgroup -# Each cgroup only contains a single GPU, which has GPU ID 0. So no longer needed to set -# one of the ranks to GPU 0 and one to GPU 1 -#if args.cuda and args.use_horovod: -# # Horovod: pin GPU to local rank. -# torch.cuda.set_device(hvd.local_rank()) +log(f"World size: {world_size}") torch.set_num_threads(int(os.environ['OMP_NUM_THREADS'])) torch.set_num_interop_threads(2) @@ -204,9 +199,10 @@ def log(s, nl=True): # Set device_type for AMP if args.cuda: - device_type="cuda" + device_type = "cuda" else: - device_type="cpu" + device_type = "cpu" + def benchmark_step(): optimizer.zero_grad() @@ -217,6 +213,7 @@ def benchmark_step(): scaler.step(optimizer) scaler.update() + log('Model: %s' % args.model) log('Batch size: %d' % args.batch_size) device = 'GPU' if args.cuda else 'CPU' From 4ddfe23d9c844f4b5880278118913ee5bbe454b1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 6 May 2024 16:52:25 +0200 Subject: [PATCH 036/119] Fix linter issues --- .../testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 44cdd7f4..68223cef 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -1,6 +1,7 @@ import reframe as rfm import reframe.utility.sanity as sn -from reframe.core.builtins import parameter, variable, run_after, sanity_function, performance_function # added only to make the linter happy +# Added only to make the linter happy +from reframe.core.builtins import parameter, variable, run_after, sanity_function, performance_function from eessi.testsuite import hooks from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM @@ -57,7 +58,7 @@ def set_tag_ci(self): @run_after('setup') def apply_setup_hooks(self): - if self.compute_device==DEVICE_TYPES[GPU]: + if self.compute_device == DEVICE_TYPES[GPU]: hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[GPU]) else: # Hybrid code, so launch 1 rank per socket. @@ -80,11 +81,11 @@ def set_ddp_env_vars(self): def filter_invalid_parameter_combinations(self): # We cannot detect this situation before the setup phase, because it requires self.num_tasks. # Thus, the core count of the node needs to be known, which is only the case after the setup phase. - msg = f"Skipping test: parallel strategy is 'None'," + msg = "Skipping test: parallel strategy is 'None'," msg += f" but requested process count is larger than one ({self.num_tasks})." self.skip_if(self.num_tasks > 1 and self.parallel_strategy is None, msg) msg = f"Skipping test: parallel strategy is {self.parallel_strategy}," - msg += f" but only one process is requested." + msg += " but only one process is requested." self.skip_if(self.num_tasks == 1 and self.parallel_strategy is not None, msg) @run_after('setup') From 73b7e846009a47436562fe5c5cbb230bf287f71c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 11:24:30 +0200 Subject: [PATCH 037/119] Can't combine generators with plus, so use chain --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 68223cef..575527cf 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -1,3 +1,5 @@ +from itertools import chain + import reframe as rfm import reframe.utility.sanity as sn # Added only to make the linter happy @@ -14,7 +16,7 @@ class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest): parallel_strategy = parameter([None, 'ddp']) compute_device = variable(str) # Both torchvision and PyTorch-bundle modules have everything needed to run this test - module_name = parameter(find_modules('torchvision') + find_modules('PyTorch-bundle')) + module_name = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle'))) descr = 'Benchmark that runs a selected torchvision model on synthetic data' From 393f270a27998a863ffb3ac5b5c57920f1faaded Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 13:25:33 +0200 Subject: [PATCH 038/119] Fix indentation of rais in check_proc_attribute_defined --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index 9357cc60..be9dec4d 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -148,4 +148,4 @@ def check_proc_attribute_defined(test: rfm.RegressionTest, attribute) -> bool: "The function utils.proc_attribute_defined should only be called after the setup() phase of ReFrame." "This is a programming error, please report this issue." ) - raise AttributeError(msg) + raise AttributeError(msg) From 829d6ff0d1ecdbabb2d35fbc32ae33e8cc50262a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 13:26:21 +0200 Subject: [PATCH 039/119] Set block distribution in sockets as well when srun is used as parallel launcher --- eessi/testsuite/hooks.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 0b73cd58..25abfa84 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -437,9 +437,11 @@ def set_compact_process_binding(test: rfm.RegressionTest): test.env_vars['I_MPI_PIN_CELL'] = 'core' # Don't bind to hyperthreads, only to physcial cores test.env_vars['I_MPI_PIN_DOMAIN'] = '%s:compact' % physical_cpus_per_task test.env_vars['OMPI_MCA_rmaps_base_mapping_policy'] = 'slot:PE=%s' % physical_cpus_per_task - # Default binding for SLURM. Only effective if the task/affinity plugin is enabled - # and when number of tasks times cpus per task equals either socket, core or thread count - test.env_vars['SLURM_CPU_BIND'] = 'verbose' + if test.current_partition.launcher_type().registered_name == 'srun': + # Set compact binding for SLURM. Only effective if the task/affinity plugin is enabled + # and when number of tasks times cpus per task equals either socket, core or thread count + test.env_vars['SLURM_DISTRIBUTION'] = 'block:block' + test.env_vars['SLURM_CPU_BIND'] = 'verbose' log(f'Set environment variable I_MPI_PIN_DOMAIN to {test.env_vars["I_MPI_PIN_DOMAIN"]}') log('Set environment variable OMPI_MCA_rmaps_base_mapping_policy to ' f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') From 23705ac380509ef3b5c81a6ef4982fe9872c472a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 16:25:24 +0200 Subject: [PATCH 040/119] Make if-statements based on launcher, and warn if an unsupported launcher is used that binding might not be effective --- eessi/testsuite/hooks.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 25abfa84..bbb527de 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -6,6 +6,7 @@ import warnings import reframe as rfm +import reframe.core.logging as rflog from eessi.testsuite.constants import * from eessi.testsuite.utils import (get_max_avail_gpus_per_node, is_cuda_required_module, log, @@ -432,20 +433,31 @@ def set_compact_process_binding(test: rfm.RegressionTest): num_cpus_per_core = test.current_partition.processor.num_cpus_per_core physical_cpus_per_task = int(test.num_cpus_per_task / num_cpus_per_core) - # Do binding for intel and OpenMPI's mpirun, and srun - # Other launchers may or may not do the correct binding - test.env_vars['I_MPI_PIN_CELL'] = 'core' # Don't bind to hyperthreads, only to physcial cores - test.env_vars['I_MPI_PIN_DOMAIN'] = '%s:compact' % physical_cpus_per_task - test.env_vars['OMPI_MCA_rmaps_base_mapping_policy'] = 'slot:PE=%s' % physical_cpus_per_task - if test.current_partition.launcher_type().registered_name == 'srun': + if test.current_partition.launcher_type().registered_name == 'mpirun': + # Do binding for intel and OpenMPI's mpirun, and srun + test.env_vars['I_MPI_PIN_CELL'] = 'core' # Don't bind to hyperthreads, only to physcial cores + test.env_vars['I_MPI_PIN_DOMAIN'] = '%s:compact' % physical_cpus_per_task + test.env_vars['OMPI_MCA_rmaps_base_mapping_policy'] = 'slot:PE=%s' % physical_cpus_per_task + log(f'Set environment variable I_MPI_PIN_CELL to {test.env_vars["I_MPI_PIN_CELL"]}') + log(f'Set environment variable I_MPI_PIN_DOMAIN to {test.env_vars["I_MPI_PIN_DOMAIN"]}') + log('Set environment variable OMPI_MCA_rmaps_base_mapping_policy to ' + f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') + elif test.current_partition.launcher_type().registered_name == 'srun': # Set compact binding for SLURM. Only effective if the task/affinity plugin is enabled # and when number of tasks times cpus per task equals either socket, core or thread count test.env_vars['SLURM_DISTRIBUTION'] = 'block:block' test.env_vars['SLURM_CPU_BIND'] = 'verbose' - log(f'Set environment variable I_MPI_PIN_DOMAIN to {test.env_vars["I_MPI_PIN_DOMAIN"]}') - log('Set environment variable OMPI_MCA_rmaps_base_mapping_policy to ' - f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') - log(f'Set environment variable SLURM_CPU_BIND to {test.env_vars["SLURM_CPU_BIND"]}') + log(f'Set environment variable SLURM_DISTRIBUTION to {test.env_vars["SLURM_DISTRIBUTION"]}') + log(f'Set environment variable SLURM_CPU_BIND to {test.env_vars["SLURM_CPU_BIND"]}') + else: + logger = rflog.getlogger() + msg = "hooks.set_compact_process_binding does not support the current launcher" + msg += f" ({test.current_partition.launcher_type().registered_name})." + msg += " The test will run, but using the default binding strategy of your parallel launcher." + msg += " This may lead to suboptimal performance." + msg += " Please expand the functionality of hooks.set_compact_process_binding for your parallel launcher." + # Warnings will, at default loglevel, be printed on stdout when executing the ReFrame command + logger.warning(msg) def set_compact_thread_binding(test: rfm.RegressionTest): From 34e78709d6dcc058599fbabe4967b2fccab70847 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 16:29:19 +0200 Subject: [PATCH 041/119] Make linter happy --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index bbb527de..cfa968f0 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -441,7 +441,7 @@ def set_compact_process_binding(test: rfm.RegressionTest): log(f'Set environment variable I_MPI_PIN_CELL to {test.env_vars["I_MPI_PIN_CELL"]}') log(f'Set environment variable I_MPI_PIN_DOMAIN to {test.env_vars["I_MPI_PIN_DOMAIN"]}') log('Set environment variable OMPI_MCA_rmaps_base_mapping_policy to ' - f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') + f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') elif test.current_partition.launcher_type().registered_name == 'srun': # Set compact binding for SLURM. Only effective if the task/affinity plugin is enabled # and when number of tasks times cpus per task equals either socket, core or thread count From 9f4666787bf816f033bb57ea8485396842a4ec2b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 7 May 2024 16:42:20 +0200 Subject: [PATCH 042/119] Make linter happy --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index cfa968f0..01b84206 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -441,7 +441,7 @@ def set_compact_process_binding(test: rfm.RegressionTest): log(f'Set environment variable I_MPI_PIN_CELL to {test.env_vars["I_MPI_PIN_CELL"]}') log(f'Set environment variable I_MPI_PIN_DOMAIN to {test.env_vars["I_MPI_PIN_DOMAIN"]}') log('Set environment variable OMPI_MCA_rmaps_base_mapping_policy to ' - f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') + f'{test.env_vars["OMPI_MCA_rmaps_base_mapping_policy"]}') elif test.current_partition.launcher_type().registered_name == 'srun': # Set compact binding for SLURM. Only effective if the task/affinity plugin is enabled # and when number of tasks times cpus per task equals either socket, core or thread count From c4d403892b69fba85bd9d6bc423482ee942df53e Mon Sep 17 00:00:00 2001 From: crivella Date: Tue, 7 May 2024 18:57:23 +0200 Subject: [PATCH 043/119] Increase time for largest test --- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index b5b3db25..c8c7b96d 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -77,6 +77,14 @@ def set_tag_ci(self): self.tags.add(TAGS['CI']) log(f'tags set to {self.tags}') + @run_after('init') + def set_increased_walltime(self): + """Increase the amount of time for the largest benchmark, so it can complete successfully.""" + max_ecut = max(QEspressoPWCheck.ecut.values) + max_nbnd = max(QEspressoPWCheck.nbnd.values) + if self.ecut == max_ecut and self.nbnd == max_nbnd: + self.time_limit = '60m' + @run_after('setup') def run_after_setup(self): """Hooks to run after the setup phase""" From d1921b4b29dbd45206a5a697df5f160d0d465680 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 7 May 2024 20:24:28 +0200 Subject: [PATCH 044/119] set SRUN_CPUS_PER_TASK --- eessi/testsuite/hooks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 0b73cd58..686f8b24 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -116,6 +116,15 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n _check_always_request_gpus(test) + if test.current_partition.launcher_type().registered_name == 'srun': + # Make sure srun inherits --cpus-per-task from the job environment for Slurm versions >= 22.05 < 23.11, + # ensuring the same task binding across all Slurm versions. + # https://bugs.schedmd.com/show_bug.cgi?id=13351 + # https://bugs.schedmd.com/show_bug.cgi?id=11275 + # https://bugs.schedmd.com/show_bug.cgi?id=15632#c43 + test.env_vars['SRUN_CPUS_PER_TASK'] = test.num_cpus_per_task + log(f'Set environment variable SRUN_CPUS_PER_TASK to {test.env_vars["SRUN_CPUS_PER_TASK"]}') + def _assign_num_tasks_per_node(test: rfm.RegressionTest, num_per: int = 1): """ From aa6cd57e323145795f28b2167cc598594180c218 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 13 May 2024 16:50:06 +0200 Subject: [PATCH 045/119] Fix https://github.com/EESSI/software-layer/issues/456#issuecomment-2107755266 until we can more permanently fix it through an LMOD hook in host_injections --- config/it4i_karolina.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py index 90062c85..2207561e 100644 --- a/config/it4i_karolina.py +++ b/config/it4i_karolina.py @@ -44,6 +44,11 @@ # Avoid https://github.com/EESSI/software-layer/issues/136 # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1) 'export OMPI_MCA_pml=ucx', + # Work around "Failed to modify UD QP to INIT on mlx5_0: Operation not permitted" issue + # until we can resolve this through an LMOD hook in host_injections. + # See https://github.com/EESSI/software-layer/issues/456#issuecomment-2107755266 + 'export OMPI_MCA_mtl="^ofi"', + 'export OMPI_MCA_btl="^ofi"', ], 'launcher': 'mpirun', # Use --export=None to avoid that login environment is passed down to submitted jobs From 955ce034dc137beea485e906ffec0c5bba94a855 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 13 May 2024 16:53:58 +0200 Subject: [PATCH 046/119] Make linter happy, explain better when this can be removed again --- config/it4i_karolina.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py index 2207561e..cdd6957e 100644 --- a/config/it4i_karolina.py +++ b/config/it4i_karolina.py @@ -46,8 +46,9 @@ 'export OMPI_MCA_pml=ucx', # Work around "Failed to modify UD QP to INIT on mlx5_0: Operation not permitted" issue # until we can resolve this through an LMOD hook in host_injections. + # (then these OMPI_MCA_btl & mtl can be removed again) # See https://github.com/EESSI/software-layer/issues/456#issuecomment-2107755266 - 'export OMPI_MCA_mtl="^ofi"', + 'export OMPI_MCA_mtl="^ofi"', 'export OMPI_MCA_btl="^ofi"', ], 'launcher': 'mpirun', From 625a6138fb049b5e56ee0fe82ef2b1978fa47aa0 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 13 May 2024 18:05:03 +0200 Subject: [PATCH 047/119] Reduce the iteration count to make the OSU tests run faster, especially on slower interconnects --- eessi/testsuite/tests/apps/osu.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eessi/testsuite/tests/apps/osu.py b/eessi/testsuite/tests/apps/osu.py index 2fab94d7..83bbd0f4 100644 --- a/eessi/testsuite/tests/apps/osu.py +++ b/eessi/testsuite/tests/apps/osu.py @@ -53,6 +53,11 @@ class EESSI_OSU_Micro_Benchmarks_pt2pt(osu_benchmark): # unset num_tasks_per_node from the hpctestlib. num_tasks_per_node = None + # Set num_warmup_iters to 5 to reduce execution time, especially on slower interconnects + num_warmup_iters = 5 + # Set num_iters to 10 to reduce execution time, especially on slower interconnects + num_iters = 10 + @run_after('init') def filter_scales_2gpus(self): """Filter out scales with < 2 GPUs if running on GPUs""" @@ -169,6 +174,11 @@ class EESSI_OSU_Micro_Benchmarks_coll(osu_benchmark): # Unset num_tasks_per_node from hpctestlib num_tasks_per_node = None + # Set num_warmup_iters to 5 to reduce execution time, especially on slower interconnects + num_warmup_iters = 5 + # Set num_iters to 10 to reduce execution time, especially on slower interconnects + num_iters = 10 + @run_after('init') def run_after_init(self): """hooks to run after init phase""" From e882e6ddeed5041059287402f3fa67107723c28e Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 14 May 2024 09:37:24 +0200 Subject: [PATCH 048/119] First commit ESPResSo test. Still WIP and needs more polishing. --- .../tests/apps/espresso/benchmarks.csv | 27 ++++ .../testsuite/tests/apps/espresso/espresso.py | 97 +++++++++++++ eessi/testsuite/tests/apps/espresso/job.sh | 10 ++ .../testsuite/tests/apps/espresso/madelung.py | 132 ++++++++++++++++++ eessi/testsuite/tests/apps/espresso/plot.py | 39 ++++++ .../apps/espresso/scripts_Espresso.tar.gz | Bin 0 -> 3089 bytes 6 files changed, 305 insertions(+) create mode 100644 eessi/testsuite/tests/apps/espresso/benchmarks.csv create mode 100644 eessi/testsuite/tests/apps/espresso/espresso.py create mode 100644 eessi/testsuite/tests/apps/espresso/job.sh create mode 100644 eessi/testsuite/tests/apps/espresso/madelung.py create mode 100644 eessi/testsuite/tests/apps/espresso/plot.py create mode 100644 eessi/testsuite/tests/apps/espresso/scripts_Espresso.tar.gz diff --git a/eessi/testsuite/tests/apps/espresso/benchmarks.csv b/eessi/testsuite/tests/apps/espresso/benchmarks.csv new file mode 100644 index 00000000..95724751 --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/benchmarks.csv @@ -0,0 +1,27 @@ +"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std" +"weak scaling",4,2,2,1,6912,2.341e-01,8.081e-03 +"strong scaling",4,2,2,1,5832,2.496e-01,9.019e-03 +"weak scaling",16,4,2,2,27648,2.417e+00,9.576e-02 +"strong scaling",16,4,2,2,5832,3.853e-02,1.991e-03 +"weak scaling",32,4,4,2,55296,4.263e+00,1.161e+00 +"strong scaling",32,4,4,2,5832,2.194e-02,7.303e-04 +"weak scaling",1,1,1,1,1728,7.655e-02,3.434e-03 +"weak scaling",2,2,1,1,3456,1.456e-01,4.679e-03 +"strong scaling",2,2,1,1,5832,3.936e-01,1.098e-02 +"strong scaling",1,1,1,1,5832,6.333e-01,1.194e-01 +"strong scaling",64,4,4,4,5832,1.910e-02,6.132e-04 +"weak scaling",1,1,1,1,1728,9.482e-02,2.956e-03 +"weak scaling",2,2,1,1,3456,2.111e-01,6.614e-03 +"strong scaling",1,1,1,1,5832,9.133e-01,2.868e-02 +"strong scaling",16,4,2,2,5832,4.285e-02,1.327e-03 +"strong scaling",64,4,4,4,5832,1.715e-02,5.776e-04 +"strong scaling",128,8,4,4,5832,1.980e-02,7.013e-04 +"weak scaling",64,4,4,4,110592,4.375e-01,1.414e-02 +"weak scaling",100,5,5,4,172800,4.450e-01,1.437e-02 +"weak scaling",128,8,4,4,221184,8.720e+00,2.753e-01 +"weak scaling",128,8,4,4,221184,8.760e+00,3.110e-01 +"weak scaling",4,2,2,1,6912,2.626e-01,8.142e-03 +"weak scaling",4,2,2,1,6912,2.780e-01,8.683e-03 +"weak scaling",4,2,2,1,6912,2.627e-01,8.391e-03 +"weak scaling",4,2,2,1,6912,2.617e-01,8.155e-03 +"weak scaling",2,2,1,1,3456,2.028e-01,6.255e-03 diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py new file mode 100644 index 00000000..494abf67 --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -0,0 +1,97 @@ +""" +This module tests Espresso in available modules containing substring 'ESPResSo' which is different from Quantum Espresso. +Tests included: +- P3M benchmark - Ionic crystals + - Weak scaling + - Strong scaling +Weak and strong scaling are options that are needed to be provided tothe script and the system is either scaled based on +number of cores or kept constant. +""" + +import reframe as rfm +from reframe.core.builtins import parameter, run_after # added only to make the linter happy +from reframe.utility import reframe + +from hpctestlib.microbenchmarks.mpi.osu import osu_benchmark + +from eessi.testsuite import hooks, utils +from eessi.testsuite.constants import * +from eessi.testsuite.utils import find_modules, log + +@rfm.simple_test +class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): + '''''' + scale = parameter(SCALES.keys()) + valid_prog_environs = ['default'] + valid_systems = ['*'] + time_limit = '30m' + # Need to check if QuantumESPRESSO also gets listed. + module_name = parameter(find_modules('ESPResSo')) + # device type is parameterized for an impending CUDA ESPResSo module. + device_type = parameter([DEVICE_TYPES[CPU]]) + + executable = 'python3 madelung.py' + + default_strong_scaling_system_size = 9 + default_weak_scaling_system_size = 6 + + benchmark_info = parameter([ + ('mpi.ionic_crystals.p3m'), + ], fmt=lambda x: x[0], loggable=True) + + + @run_after('init') + def run_after_init(self): + """hooks to run after init phase""" + + # Filter on which scales are supported by the partitions defined in the ReFrame configuration + hooks.filter_supported_scales(self) + + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + + hooks.set_modules(self) + + # Set scales as tags + hooks.set_tag_scale(self) + + @run_after('init') + def set_tag_ci(self): + """ Setting tests under CI tag. """ + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']): + self.tags.add('CI') + log(f'tags set to {self.tags}') + + if (self.benchmark_info[0] == 'mpi.ionic_crystals.p3m'): + self.tags.add('ionic_crystals_p3m') + + + @run_after('init') + def set_mem(self): + """ Setting an extra job option of memory. """ + self.extra_resources = {'memory': {'size': '50GB'}} + + @run_after('init') + def set_executable_opts(self): + """Set executable opts based on device_type parameter""" + num_default = 0 # If this test already has executable opts, they must have come from the command line + hooks.check_custom_executable_opts(self, num_default=num_default) + if not self.has_custom_executable_opts: + # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a + # corresponding min node size has to be chozen. + self.executable_opts += ['--size', self.default_weak_scaling_system_size, '--weak-scaling'] + utils.log(f'executable_opts set to {self.executable_opts}') + + @run_after('setup') + def set_num_tasks_per_node(self): + """ Setting number of tasks per node and cpus per task in this function. This function sets num_cpus_per_task + for 1 node and 2 node options where the request is for full nodes.""" + hooks.assign_tasks_per_compute_unit(self, COMPUTE_UNIT[CPU]) + + @sanity_function + def assert_sanity(self): + '''Check all sanity criteria''' + return sn.all([ + self.assert_completion(), + self.assert_convergence(), + ]) + diff --git a/eessi/testsuite/tests/apps/espresso/job.sh b/eessi/testsuite/tests/apps/espresso/job.sh new file mode 100644 index 00000000..17399c52 --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/job.sh @@ -0,0 +1,10 @@ +#!/bin/bash +#SBATCH --time=00:40:00 +#SBATCH --output %j.stdout +#SBATCH --error %j.stderr +module load spack/default gcc/12.3.0 cuda/12.3.0 openmpi/4.1.6 \ + fftw/3.3.10 boost/1.83.0 python/3.12.1 +source ../espresso-4.3/venv/bin/activate +srun --cpu-bind=cores python3 madelung.py --size 6 --weak-scaling +srun --cpu-bind=cores python3 madelung.py --size 9 --strong-scaling +deactivate diff --git a/eessi/testsuite/tests/apps/espresso/madelung.py b/eessi/testsuite/tests/apps/espresso/madelung.py new file mode 100644 index 00000000..4bfb1df1 --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/madelung.py @@ -0,0 +1,132 @@ +# +# Copyright (C) 2013-2024 The ESPResSo project +# +# This file is part of ESPResSo. +# +# ESPResSo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ESPResSo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import espressomd +import espressomd.version +import espressomd.electrostatics +import argparse +import pathlib +import time +import numpy as np + +parser = argparse.ArgumentParser(description="Benchmark P3M simulations.") +parser.add_argument("--size", metavar="S", action="store", + default=9, required=False, type=int, + help="Problem size, such that the number of particles N is " + "equal to (2*S)^2; with --weak-scaling this number N " + "is multiplied by the number of cores!") +parser.add_argument("--gpu", action=argparse.BooleanOptionalAction, + default=False, required=False, help="Use GPU implementation") +parser.add_argument("--topology", metavar=("X", "Y", "Z"), nargs=3, action="store", + default=None, required=False, type=int, help="Cartesian topology") +parser.add_argument("--output", metavar="FILEPATH", action="store", + type=str, required=False, default="benchmarks.csv", + help="Output file (default: benchmarks.csv)") +group = parser.add_mutually_exclusive_group() +group.add_argument("--weak-scaling", action="store_true", + help="Weak scaling benchmark (Gustafson's law: constant work per core)") +group.add_argument("--strong-scaling", action="store_true", + help="Strong scaling benchmark (Amdahl's law: constant total work)") +args = parser.parse_args() + +def get_reference_values_per_ion(base_vector): + madelung_constant = -1.74756459463318219 + base_tensor = base_vector * np.eye(3) + ref_energy = madelung_constant + ref_pressure = madelung_constant * base_tensor / np.trace(base_tensor) + return ref_energy, ref_pressure + +def get_normalized_values_per_ion(system): + energy = system.analysis.energy()["coulomb"] + p_scalar = system.analysis.pressure()["coulomb"] + p_tensor = system.analysis.pressure_tensor()["coulomb"] + N = len(system.part) + V = system.volume() + return 2. * energy / N, 2. * p_scalar * V / N, 2. * p_tensor * V / N + +# initialize system +system = espressomd.System(box_l=[100., 100., 100.]) +system.time_step = 0.01 +system.cell_system.skin = 0.4 + +# set MPI Cartesian topology +node_grid = system.cell_system.node_grid.copy() +n_cores = int(np.prod(node_grid)) +if args.topology: + system.cell_system.node_grid = node_grid = args.topology + +# place ions on a cubic lattice +base_vector = np.array([1., 1., 1.]) +lattice_size = 3 * [2 * args.size] +if args.weak_scaling: + lattice_size = np.multiply(lattice_size, node_grid) +system.box_l = np.multiply(lattice_size, base_vector) +for j in range(lattice_size[0]): + for k in range(lattice_size[1]): + for l in range(lattice_size[2]): + _ = system.part.add(pos=np.multiply([j, k, l], base_vector), + q=(-1.)**(j + k + l), fix=3 * [True]) + +# setup P3M algorithm +algorithm = espressomd.electrostatics.P3M +if args.gpu: + algorithm = espressomd.electrostatics.P3MGPU +solver = algorithm(prefactor=1., accuracy=1e-6) +if (espressomd.version.major(), espressomd.version.minor()) == (4, 2): + system.actors.add(solver) +else: + system.electrostatics.solver = solver + +# run checks +forces = np.copy(system.part.all().f) +energy, p_scalar, p_tensor = get_normalized_values_per_ion(system) +ref_energy, ref_pressure = get_reference_values_per_ion(base_vector) +np.testing.assert_allclose(energy, ref_energy, atol=1e-12, rtol=5e-6) +np.testing.assert_allclose(p_scalar, np.trace(ref_pressure) / 3., + atol=1e-12, rtol=2e-5) +np.testing.assert_allclose(p_tensor, ref_pressure, atol=1e-12, rtol=2e-5) +np.testing.assert_allclose(forces, 0., atol=1e-5, rtol=0.) +np.testing.assert_allclose(np.median(np.abs(forces)), 0., atol=2e-6, rtol=0.) + + +print("Executing sanity ...\n") +print (np.all([np.allclose(energy, ref_energy, atol=1e-12, rtol=5e-6), + np.allclose(p_scalar, np.trace(ref_pressure) / 3., + atol=1e-12, rtol=2e-5), + np.allclose(p_tensor, ref_pressure, atol=1e-12, rtol=2e-5), + np.allclose(forces, 0., atol=1e-5, rtol=0.), + np.allclose(np.median(np.abs(forces)), 0., atol=2e-6, rtol=0.)])) + +print("Sanity checking ...\n") +# sample runtime +n_steps = 10 +timings = [] +for _ in range(10): + tick = time.time() + system.integrator.run(n_steps) + tock = time.time() + timings.append((tock - tick) / n_steps) + +# write results to file +header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std"\n' +report = f'"{"weak scaling" if args.weak_scaling else "strong scaling"}",{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},{np.mean(timings):.3e},{np.std(timings, ddof=1):.3e}\n' +if pathlib.Path(args.output).is_file(): + header = "" +with open(args.output, "a") as f: + f.write(header + report) diff --git a/eessi/testsuite/tests/apps/espresso/plot.py b/eessi/testsuite/tests/apps/espresso/plot.py new file mode 100644 index 00000000..c9a023c4 --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/plot.py @@ -0,0 +1,39 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.ticker as mtick + +df = pd.read_csv("benchmarks.csv") +df = df.sort_values(by=["mode", "cores", "mpi.x", "mpi.y", "mpi.z"]) + +group = df.query(f"mode == 'strong scaling'") + +fig = plt.figure(figsize=(12, 6)) +ax = fig.subplots().axes +xdata = group["cores"].to_numpy() +ydata = group["mean"].to_numpy() +ax.axline((xdata[0], xdata[0]), slope=1, linestyle="--", color="grey", label="Theoretical maximum") +ax.plot(xdata, ydata[0] / ydata, "o-", label="Measurements") +ax.set_title("Strong scaling") +ax.set_xlabel("Number of cores") +ax.set_ylabel("Speed-up") +ax.set_xscale("log", base=2) +ax.set_yscale("log", base=10) +ax.legend() +plt.show() + +group = df.query(f"mode == 'weak scaling'") + +fig = plt.figure(figsize=(12, 6)) +ax = fig.subplots().axes +xdata = group["cores"].to_numpy() +ydata = group["mean"].to_numpy() +ax.axline((-np.inf, 1), slope=0, linestyle="--", color="grey", label="Theoretical maximum") +ax.plot(xdata, ydata[0] / ydata, "o-", label="Measurements") +ax.set_title("Weak scaling") +ax.set_xlabel("Number of cores") +ax.set_ylabel("Efficiency") +ax.set_xscale("log", base=2) +ax.yaxis.set_major_formatter(mtick.PercentFormatter(1)) +ax.legend() +plt.show() diff --git a/eessi/testsuite/tests/apps/espresso/scripts_Espresso.tar.gz b/eessi/testsuite/tests/apps/espresso/scripts_Espresso.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..24e2621fec80e082c1830617209e16fa63c8df4e GIT binary patch literal 3089 zcmV+s4DRzEiwFP!000001MM1XZ`(MspYRr0(9qv~LhNs*Q44Y*X-3GCt9eGlNARK` z{T}LkZ+NObC9il|WYK&fiSf!LuI+ez*LHn!zF_3_*~uvrXPl%NzhZ&Zkl=g~2{Maf zM&O##OcFk8FfE1B&>?2V7!iCXZ)wK%Nx^dx(1c_xjD*ahX)b}0Bs2*JJR?gUMzaD~ zz)+rqEF%*1kStpY6oU_Mjz5w&EMXaq$w@wqqkz1N0+t9yC@6#@V!=W(MK(xqh#H(# z8jwQ{e5s6h(jzQ__ZhiinSf{F)gmf>J;F2KVNi)`XN0FnW`eE-iK$#sZq9l&xsXJO zV!Gfd%wz$~U~acj9Fr*{xnQ$A?g2I6k{^%G-+uUbP7aPgksl9EPY;gIKm8qW<$}Xo zcEd_e(K3xAU<~ugXd(+x8yLKQefsJxP#nBGdUtgG33MliN9V_{&(6rh_82K0~Eb*4N6#tWIGQ7p>&egant&@2wgg5EH2X~3cz z(2J4)g6hlu0v`nkTu%v-uz{ zd5FP3Q5y|x!XXryNHZn`Uxu3_R^_cNZ&(b`opHzpG73bEK(jf-H|SYCr&KQDXj(tY zXxY3<@+Hg|;x9=xO%Y{e+%T{XvU$E_i9AtnjF5>Si&Qj?^_MIO7E794lN0Yf5z#V_ zRd@XSQLGj?3JXn+gjcx`DOkvXCbj%4+5HmrJ zaVh9q87k002Z##EiQzs!Gym(B!A|=9TSl+@BA{^u;h`d<;(z=oKfnX?$*5c()8d|H z0a!--@UU6uX}&UKvb#AIvE3HrK^&+f5+&sXiRQ-cerz}=R#bBSTGcaK`^Ni;y ztk>48mbrw`jEf1o3*uZrUYjT^qr%>pdOKq8Pj(`+{DB#liv9@1qz?B+KVrPW24yCA z@=QR*x!s2dPGBT~0>a^a3So^g-OOP#Q9xEp=D%C7Gey2lvx8+w7x9K#l1o_7s2u8q zLAf%kx}bIfjHrP-$ehVZ#%3%7zHD+s21ON% z9E#u;B;eVWH$Vw8qX9Eo!?iS-XGu$FZ;kuPFbU6=VD_I`c;7Oj5Rxq`qcobAqZWk} zRftGfdchYPNNv*^-qQ;aD5mgwQw8uacvs!Ho16i8lUn_1m zJcgJx)=cDDVAg7S19IFe2O9OC1ATkAQm`7mb&1(xgDKJYiQMC)OAu}5gUC=0nMuTRWS~SCULs+$lGCCgP4b0m2Yt*QI8bbgR z>_iAO$I#T}Q|R5GIf3fVwDyV`W4vQCS{N4&8czKK(^be5Y#0E`1Mgh8@S|km;ANvX z=Fdr$XiDAh5rc@TwiibGRc|$m##hRjkJ7ZWi-!H}753@SPS4pI=7oJ(Ei^>BegMg- zk!r;F0YtZrh%i|N04@wn;6}>DxUKocRgYZvNPM}j_V&uxhR&T)*9M1vrQTw07B)*q@sH#VDkYe}y#`Si9l{MIr`@SRDzhR@Q`k?OBgT^M*{9^>@A?voz(pUAAcdMNXwNdju;&O}$eQ+t#C`xQJN@i68Ii zR8+i7v&9YM4_n-DF;d(ZU_7ZEW}=D?Mw4S}Fam<`;o?%It;s6UICd4*5TVzAfz+yM zRb{mbLMR{%_SoQh3q%+d`w~U+t(28ii>4_{Lc>sKeI*B-*RTNPZ(#-FQGkHtC2(hl z&9}B-G*rvwnZAT%t@m_Q&fvOCBkQiZ7S;7=^6J4zJvih%j@0Y5o{t9<>34wgS_UNoXy7+j%nL!NnH2Ar)l(}zt ztQ-cd>rvRmgnTx3%C|U`XjUI8SSN5blpP)ubH@1i`LFFbzIX5Z*Y#Y#JOBM2@P95wZGG|e z{GJPcPH#AZ?CM&c51G<-dV7|=hj*TKe+@PiJ9{2d_@kks7(shIDk;__IKzs>-5vUS zNa5_V7q$%)JG)5hZpvtAln~F_+wl;w=UAhWvrPzK`HEy`#~lH&{_0U;{1)7)PK9~9$*c3b`-v6`JTT`snSkP&+~VNpa}dbOZnDtceLHM zHAOY;kylbUmOa{gz*5zpq8M79=T#KtY@AJsq2DY26bV?=vDGw&mgBi<+JAJ2ckl?L zF_+)++aJOg^1thO8{hwI*Xi>A_kb-QFPcvsNeBb@L9O|cvZN9N2Ah?%z&rIimRsIq zX9yh!ykFw2riC*+2MnzYT9FC#8pi!&T(d+8XRx(`7P{Q87}H{WQ57jtS1D4LDN@%d zQWYxP%hpG^!02<%vci}tdffRvThCt4K&Cc}=BQXKEx5yzFSy`FeQe+cVQ88fy~9p3 znp+~D;`9V;-sl|@+FeK`#f=t9yr{Ihv?QM>dqV46tYfIkxrRXRfEDP%3`22)=CeoY zJJ=+M7<#<116zN<5weKcSnu~?!U2zAbF9xZhSQJfl*MEHe8E5mus4mt7kAMzU+PK* zs$B}`k)q-bZF^CU=)Aw;@t#rKC*kq2D2WB^BV{C^550F9yn217hzb2+{bdxR*N z^@!6r-2SsTy!~0?cewlYY!(F(tn=b49A3Z#ghEj#Ig=S4en`nOL!}Vwgu&K1k%!H& zV?G!Pe{^2`tH2eXg2}$Ej4#xGd+6M+|2DMpUH$(a@brhlG)e}r3K!bbvzG_wuilb= z9}j-Uw!QD$`>@qnjhCkk3G$bOW!S$f}{7U9Q zFmPaVXxSvlLt5YCVFpz3fp0n1ko>fE7&x2B+kppYj!mYV3psGCJtRsCx!?)B0w#_o zcn&FoSk|CE0`B{kH@IQRjhZHY`n;i%X#$UG`~4uz`!F7k%TFsMGq3r$2Tcws|EC2R z-uoX3zn#N~-2WD Date: Tue, 14 May 2024 22:15:57 +0200 Subject: [PATCH 049/119] Moved some files and added __init__.py to mark the directories and sub-dirs as part of the package. Sanity for the weak scaling is already added. Next step is to extract the performance timing and log it. --- .../testsuite/tests/apps/espresso/__init__.py | 0 .../tests/apps/espresso/benchmarks.csv | 7 +++ .../testsuite/tests/apps/espresso/espresso.py | 19 +++++++- .../tests/apps/espresso/src/__init__.py | 0 .../tests/apps/espresso/{ => src}/job.sh | 0 .../tests/apps/espresso/{ => src}/madelung.py | 44 +++++++++++++----- .../tests/apps/espresso/{ => src}/plot.py | 0 .../{ => src}/scripts_Espresso.tar.gz | Bin 8 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 eessi/testsuite/tests/apps/espresso/__init__.py create mode 100644 eessi/testsuite/tests/apps/espresso/src/__init__.py rename eessi/testsuite/tests/apps/espresso/{ => src}/job.sh (100%) rename eessi/testsuite/tests/apps/espresso/{ => src}/madelung.py (76%) rename eessi/testsuite/tests/apps/espresso/{ => src}/plot.py (100%) rename eessi/testsuite/tests/apps/espresso/{ => src}/scripts_Espresso.tar.gz (100%) diff --git a/eessi/testsuite/tests/apps/espresso/__init__.py b/eessi/testsuite/tests/apps/espresso/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/eessi/testsuite/tests/apps/espresso/benchmarks.csv b/eessi/testsuite/tests/apps/espresso/benchmarks.csv index 95724751..9091534b 100644 --- a/eessi/testsuite/tests/apps/espresso/benchmarks.csv +++ b/eessi/testsuite/tests/apps/espresso/benchmarks.csv @@ -25,3 +25,10 @@ "weak scaling",4,2,2,1,6912,2.627e-01,8.391e-03 "weak scaling",4,2,2,1,6912,2.617e-01,8.155e-03 "weak scaling",2,2,1,1,3456,2.028e-01,6.255e-03 +"weak scaling",2,2,1,1,3456,3.247e-01,1.026e-02 +"weak scaling",2,2,1,1,3456,3.249e-01,1.029e-02 +"weak scaling",2,2,1,1,3456,3.257e-01,1.028e-02 +"weak scaling",2,2,1,1,3456,3.375e-01,1.095e-02 +"weak scaling",2,2,1,1,3456,3.367e-01,1.086e-02 +"weak scaling",2,2,1,1,3456,3.241e-01,1.048e-02 +"weak scaling",2,2,1,1,3456,3.243e-01,1.038e-02 diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 494abf67..37f81344 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -9,6 +9,8 @@ """ import reframe as rfm +import reframe.utility.sanity as sn + from reframe.core.builtins import parameter, run_after # added only to make the linter happy from reframe.utility import reframe @@ -36,7 +38,7 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): default_weak_scaling_system_size = 6 benchmark_info = parameter([ - ('mpi.ionic_crystals.p3m'), + ('mpi.ionic_crystals.p3m', 'p3m'), ], fmt=lambda x: x[0], loggable=True) @@ -78,7 +80,7 @@ def set_executable_opts(self): if not self.has_custom_executable_opts: # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a # corresponding min node size has to be chozen. - self.executable_opts += ['--size', self.default_weak_scaling_system_size, '--weak-scaling'] + self.executable_opts += ['--size', str(self.default_weak_scaling_system_size), '--weak-scaling'] utils.log(f'executable_opts set to {self.executable_opts}') @run_after('setup') @@ -87,6 +89,19 @@ def set_num_tasks_per_node(self): for 1 node and 2 node options where the request is for full nodes.""" hooks.assign_tasks_per_compute_unit(self, COMPUTE_UNIT[CPU]) + @deferrable + def assert_completion(self): + '''Check completion''' + cao = sn.extractsingle(r'^resulting parameters:.*cao: (?P\S+),', self.stdout, 'cao', int) + return (sn.assert_found(r'^Algorithm executed.', self.stdout) and cao) + + @deferrable + def assert_convergence(self): + '''Check convergence''' + check_string = sn.assert_found(r'Final convergence met with tolerances:', self.stdout) + energy = sn.extractsingle(r'^\s+energy:\s+(?P\S+)', self.stdout, 'energy', float) + return (check_string and (energy != 0.0)) + @sanity_function def assert_sanity(self): '''Check all sanity criteria''' diff --git a/eessi/testsuite/tests/apps/espresso/src/__init__.py b/eessi/testsuite/tests/apps/espresso/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/eessi/testsuite/tests/apps/espresso/job.sh b/eessi/testsuite/tests/apps/espresso/src/job.sh similarity index 100% rename from eessi/testsuite/tests/apps/espresso/job.sh rename to eessi/testsuite/tests/apps/espresso/src/job.sh diff --git a/eessi/testsuite/tests/apps/espresso/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py similarity index 76% rename from eessi/testsuite/tests/apps/espresso/madelung.py rename to eessi/testsuite/tests/apps/espresso/src/madelung.py index 4bfb1df1..628d8eab 100644 --- a/eessi/testsuite/tests/apps/espresso/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -93,27 +93,46 @@ def get_normalized_values_per_ion(system): else: system.electrostatics.solver = solver + +print("Algorithm executed. \n") + +atol_energy = atol_pressure = 1e-12 +atol_forces = 1e-5 +atol_abs_forces = 2e-6 + +rtol_energy = 5e-6 +rtol_pressure = 2e-5 +rtol_forces = 0. +rtol_abs_forces = 0. # run checks forces = np.copy(system.part.all().f) energy, p_scalar, p_tensor = get_normalized_values_per_ion(system) ref_energy, ref_pressure = get_reference_values_per_ion(base_vector) -np.testing.assert_allclose(energy, ref_energy, atol=1e-12, rtol=5e-6) +np.testing.assert_allclose(energy, ref_energy, atol=atol_energy, rtol=rtol_energy) np.testing.assert_allclose(p_scalar, np.trace(ref_pressure) / 3., - atol=1e-12, rtol=2e-5) -np.testing.assert_allclose(p_tensor, ref_pressure, atol=1e-12, rtol=2e-5) -np.testing.assert_allclose(forces, 0., atol=1e-5, rtol=0.) -np.testing.assert_allclose(np.median(np.abs(forces)), 0., atol=2e-6, rtol=0.) + atol=atol_pressure, rtol=rtol_pressure) +np.testing.assert_allclose(p_tensor, ref_pressure, atol=atol_pressure, rtol=rtol_pressure) +np.testing.assert_allclose(forces, 0., atol=atol_forces, rtol=rtol_forces) +np.testing.assert_allclose(np.median(np.abs(forces)), 0., atol=atol_abs_forces, rtol=rtol_abs_forces) -print("Executing sanity ...\n") -print (np.all([np.allclose(energy, ref_energy, atol=1e-12, rtol=5e-6), +print("Executing sanity checks...\n") +if (np.all([np.allclose(energy, ref_energy, atol=atol_energy, rtol=rtol_energy), np.allclose(p_scalar, np.trace(ref_pressure) / 3., - atol=1e-12, rtol=2e-5), - np.allclose(p_tensor, ref_pressure, atol=1e-12, rtol=2e-5), - np.allclose(forces, 0., atol=1e-5, rtol=0.), - np.allclose(np.median(np.abs(forces)), 0., atol=2e-6, rtol=0.)])) + atol=atol_pressure, rtol=rtol_pressure), + np.allclose(p_tensor, ref_pressure, atol=atol_pressure, rtol=rtol_pressure), + np.allclose(forces, 0., atol=atol_forces, rtol=rtol_forces), + np.allclose(np.median(np.abs(forces)), 0., atol=atol_abs_forces, rtol=rtol_abs_forces)])): + print("Final convergence met with tolerances: \n\ + energy: ", atol_energy, "\n\ + p_scalar: ", atol_pressure, "\n\ + p_tensor: ", atol_pressure, "\n\ + forces: ", atol_forces, "\n\ + abs_forces: ", atol_abs_forces, "\n") +else: + print("At least one parameter did not meet the tolerance, see the log above.\n") -print("Sanity checking ...\n") +print("Sampling runtime...\n") # sample runtime n_steps = 10 timings = [] @@ -126,6 +145,7 @@ def get_normalized_values_per_ion(system): # write results to file header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std"\n' report = f'"{"weak scaling" if args.weak_scaling else "strong scaling"}",{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},{np.mean(timings):.3e},{np.std(timings, ddof=1):.3e}\n' +print(report) if pathlib.Path(args.output).is_file(): header = "" with open(args.output, "a") as f: diff --git a/eessi/testsuite/tests/apps/espresso/plot.py b/eessi/testsuite/tests/apps/espresso/src/plot.py similarity index 100% rename from eessi/testsuite/tests/apps/espresso/plot.py rename to eessi/testsuite/tests/apps/espresso/src/plot.py diff --git a/eessi/testsuite/tests/apps/espresso/scripts_Espresso.tar.gz b/eessi/testsuite/tests/apps/espresso/src/scripts_Espresso.tar.gz similarity index 100% rename from eessi/testsuite/tests/apps/espresso/scripts_Espresso.tar.gz rename to eessi/testsuite/tests/apps/espresso/src/scripts_Espresso.tar.gz From 6f2c2db13341c6763fea1e0e106c72e410b15c85 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 11:53:19 +0200 Subject: [PATCH 050/119] Add available memory for one of the Snellius partitions, for testing --- config/surf_snellius.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/surf_snellius.py b/config/surf_snellius.py index 9e4ee269..542f3ee2 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -72,6 +72,9 @@ 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + 'mem_per_node': 336 + }, 'descr': 'AMD Genoa CPU partition with native EESSI stack' }, From d1c7f74656c9e5f4a340fb5a854f3b2aa9f5672b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 11:53:53 +0200 Subject: [PATCH 051/119] Implement hook for requesting memory from the scheduler --- eessi/testsuite/hooks.py | 75 ++++++++++++++++++- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 5 ++ eessi/testsuite/utils.py | 38 +++++++++- 3 files changed, 115 insertions(+), 3 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index c06ff572..dd7ea2bb 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -9,7 +9,7 @@ from eessi.testsuite.constants import * from eessi.testsuite.utils import (get_max_avail_gpus_per_node, is_cuda_required_module, log, - check_proc_attribute_defined) + check_proc_attribute_defined, check_extras_key_defined) def _assign_default_num_cpus_per_node(test: rfm.RegressionTest): @@ -373,6 +373,79 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic log(f'valid_systems set to {test.valid_systems}') +## TODO: function should take everything in MB, as schedulers (at least slurm) does not except asking for fractional memory +## ie --mem=7.0G is invalid. This should be done as --mem=7168M. +## It's probably better if this function does everything in MB, and the ReFrame config also specifies available mem per node in MB. +## Then, we should make sure the numbers are integers by rounding up for app_mem_req (1 MB more should never really be an issue) +## and probably down for the default_mem (as to not ask for more than the equivalent share of a core) +def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): + """ + This hook will request a specific amount of memory per node to the batch scheduler. + First, it computes which fraction of CPUs is requested from a node, and how much the corresponding (proportional) + amount of memory would be. + Then, the hook compares this to how much memory the application claims to need per node (app_mem_req). + It then passes the maximum of these two numbers to the batch scheduler as a memory request. + + Note: using this hook requires that the ReFrame configuration defines system.partition.extras['mem_per_node'] + + Arguments: + - test: the ReFrame test to which this hook should apply + - app_mem_req: the amount of memory this application needs (per node) in gigabytes + + Example 1: + - A system with 128 cores per node, 64 GB mem per node is used. + - The test is launched on 64 cores + - The app_mem_req is 40 (GB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32 GB. + The app_mem_req is higher. Thus, 40GB (per node) is requested from the batch scheduler. + + Example 2: + - A system with 128 cores per node, 128 GB mem per node is used. + - The test is launched on 64 cores + - the app_mem_req is 40 (GB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64 GB. + This is higher than the app_mem_req. Thus, 64 GB (per node) is requested from the batch scheduler. + """ + # Check that the systems.partitions.extra dict in the ReFrame config contains mem_per_node + check_extras_key_defined(test, 'mem_per_node') + + # Skip if the current partition doesn't have sufficient memory to run the application + msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} GB" + msg += " memory available (per node) accodring to the current ReFrame configuration," + msg += f" but {app_mem_req} GB is needed" + test.skip_if(test.current_partition.extras['mem_per_node'] < app_mem_req, msg) + + # Compute what is higher: the requested memory, or the memory available proportional to requested CPUs + # Fraction of CPU cores requested + check_proc_attribute_defined(test, 'num_cpus') + cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus + default_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] + + # Request the maximum of the default_mem, and app_mem_req to the scheduler + req_mem_per_node = max(default_mem, app_mem_req) + if test.current_partition.scheduler.registered_name == 'slurm' or test.current_partition.scheduler.registered_name == 'squeue': + # SLURMs --mem defines memory per node, see https://slurm.schedmd.com/sbatch.html + test.extra_resources = {'memory': {'size': '%sG' % req_mem_per_node }} + log(f"Requested {req_mem_per_node}GB per node from the SLURM batch scheduler") + elif test.current_partition.scheduler.registered_name == 'torque': + # Torque/moab requires asking for --pmem (--mem only works single node and thus doesnt generalize) + # See https://docs.adaptivecomputing.com/10-0-1/Torque/torque.htm#topics/torque/3-jobs/3.1.3-requestingRes.htm + req_mem_per_task = req_mem_per_node / test.num_tasks_per_node + # We assume here the reframe config defines the extra resource memory as asking for pmem + # i.e. 'options': ['--pmem={size}'] + test.extra_resources = {'memory': {'size': '%sgb' % req_mem_per_task }} + log(f"Requested {req_mem_per_task}GB per task from the torque batch scheduler") + else: + logger = rflog.getlogger() + msg = "hooks.req_memory_per_node does not support the scheduler you configured" + msg += f" ({test.current_partition.scheduler.registered_name})." + msg += " The test will run, but since it doesn't request the required amount of memory explicitely," + msg += " it may result in an out-of-memory error." + msg += " Please expand the functionality of hooks.req_memory_per_node for your scheduler." + # Warnings will, at default loglevel, be printed on stdout when executing the ReFrame command + logger.warning(msg) + + def set_modules(test: rfm.RegressionTest): """ Skip current test if module_name is not among a list of modules, diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index c8c7b96d..b98807d5 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -97,6 +97,11 @@ def run_after_setup(self): else: hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[CPU]) + @run_after('setup') + def request_mem(self): + memory_required = self.num_tasks_per_node * 1 + 2 + hooks.req_memory_per_node(test=self, app_mem_req=memory_required) + @run_after('setup') def set_omp_num_threads(self): """ diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index 9357cc60..ee679295 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -145,7 +145,41 @@ def check_proc_attribute_defined(test: rfm.RegressionTest, attribute) -> bool: else: msg = ( "This test's current_partition is not set yet. " - "The function utils.proc_attribute_defined should only be called after the setup() phase of ReFrame." + "The function utils.check_proc_attribute_defined should only be called after the setup() phase of ReFrame." "This is a programming error, please report this issue." ) - raise AttributeError(msg) + raise AttributeError(msg) + + +def check_extras_key_defined(test: rfm.RegressionTest, extra_key) -> bool: + """ + Checks if a specific key is defined in the 'extras' dictionary for the current partition + (i.e. if test.current_partition.extras[extra_key] is defined) + If not, throws an informative error message. + Note that partition extras are defined by free text keys, so any string is (potentially) valid. + + Arguments: + - test: the reframe regression test instance for which should be checked if the key is defined in 'extras' + - extra_key: key for which to check in the 'extras' dictionary + + Return: + - True (bool) if the key is defined + - Function does not return (but raises an error) if the attribute is undefined + """ + + if test.current_partition: + if extra_key in test.current_partition.extras: + return True + else: + msg = ( + f"Key '{extra_key}' missing in the 'extras' dictionary for partition '{test.current_partition.name}'." + "Please define this key for the relevant partition in the ReFrame configuration file (see " + "https://reframe-hpc.readthedocs.io/en/stable/config_reference.html#config.systems.partitions.extras)." + ) + else: + msg = ( + "This test's current_partition is not set yet. " + "The function utils.check_extras_key_defined should only be called after the setup() phase of ReFrame." + "This is a programming error, please report this issue." + ) + raise AttributeError(msg) From 4242927ff621066d4429a48e9e22582685762789 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 13:47:07 +0200 Subject: [PATCH 052/119] Convert to MB before requesting the scheduler, as schedulers (at least slurm) don't accept fractional memory requests. Rounding in MB we introduce less error --- eessi/testsuite/hooks.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index dd7ea2bb..abac264e 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -377,7 +377,7 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic ## ie --mem=7.0G is invalid. This should be done as --mem=7168M. ## It's probably better if this function does everything in MB, and the ReFrame config also specifies available mem per node in MB. ## Then, we should make sure the numbers are integers by rounding up for app_mem_req (1 MB more should never really be an issue) -## and probably down for the default_mem (as to not ask for more than the equivalent share of a core) +## and probably down for the proportional_mem (as to not ask for more than the equivalent share of a core) def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): """ This hook will request a specific amount of memory per node to the batch scheduler. @@ -390,10 +390,10 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): Arguments: - test: the ReFrame test to which this hook should apply - - app_mem_req: the amount of memory this application needs (per node) in gigabytes + - app_mem_req: the amount of memory this application needs (per node) in megabytes Example 1: - - A system with 128 cores per node, 64 GB mem per node is used. + - A system with 128 cores and 64 GB per node. - The test is launched on 64 cores - The app_mem_req is 40 (GB) In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32 GB. @@ -419,22 +419,29 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): # Fraction of CPU cores requested check_proc_attribute_defined(test, 'num_cpus') cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus - default_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] + proportional_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] - # Request the maximum of the default_mem, and app_mem_req to the scheduler - req_mem_per_node = max(default_mem, app_mem_req) + # First convert to MB and round - schedulers typically don't allow fractional numbers + # (and we want to reduce roundoff error, hence MB) + # Round up for app_mem_req to be on the save side: + app_mem_req = math.ceil(1024 * app_mem_req) + # Round down for proportional_mem, so we don't ask more than what is available per node + proportional_mem = math.floor(1024 * proportional_mem) + + # Request the maximum of the proportional_mem, and app_mem_req to the scheduler + req_mem_per_node = max(proportional_mem, app_mem_req) if test.current_partition.scheduler.registered_name == 'slurm' or test.current_partition.scheduler.registered_name == 'squeue': # SLURMs --mem defines memory per node, see https://slurm.schedmd.com/sbatch.html - test.extra_resources = {'memory': {'size': '%sG' % req_mem_per_node }} - log(f"Requested {req_mem_per_node}GB per node from the SLURM batch scheduler") + test.extra_resources = {'memory': {'size': '%sM' % req_mem_per_node }} + log(f"Requested {req_mem_per_node}MB per node from the SLURM batch scheduler") elif test.current_partition.scheduler.registered_name == 'torque': # Torque/moab requires asking for --pmem (--mem only works single node and thus doesnt generalize) # See https://docs.adaptivecomputing.com/10-0-1/Torque/torque.htm#topics/torque/3-jobs/3.1.3-requestingRes.htm req_mem_per_task = req_mem_per_node / test.num_tasks_per_node # We assume here the reframe config defines the extra resource memory as asking for pmem # i.e. 'options': ['--pmem={size}'] - test.extra_resources = {'memory': {'size': '%sgb' % req_mem_per_task }} - log(f"Requested {req_mem_per_task}GB per task from the torque batch scheduler") + test.extra_resources = {'memory': {'size': '%smb' % req_mem_per_task }} + log(f"Requested {req_mem_per_task}MB per task from the torque batch scheduler") else: logger = rflog.getlogger() msg = "hooks.req_memory_per_node does not support the scheduler you configured" From 4b87213947bdda8b5c1391997dcab11ddd0b0a01 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 16:27:51 +0200 Subject: [PATCH 053/119] Make sure all user defined inputs are in base-2, i.e. gibibytes etc. SLURM takes gigabytes (base-10). Torque supposedly takes mebibytes (base-2). --- config/izum_vega.py | 5 ++++ eessi/testsuite/hooks.py | 57 ++++++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/config/izum_vega.py b/config/izum_vega.py index 4c67792b..77d75796 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -59,6 +59,11 @@ 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 238.418 # in GiB + }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, { diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index abac264e..a97e7d45 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -387,32 +387,33 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): It then passes the maximum of these two numbers to the batch scheduler as a memory request. Note: using this hook requires that the ReFrame configuration defines system.partition.extras['mem_per_node'] + That field should be defined in GiB Arguments: - test: the ReFrame test to which this hook should apply - - app_mem_req: the amount of memory this application needs (per node) in megabytes + - app_mem_req: the amount of memory this application needs (per node) in GiB Example 1: - - A system with 128 cores and 64 GB per node. + - A system with 128 cores and 64 GiB per node. - The test is launched on 64 cores - - The app_mem_req is 40 (GB) - In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32 GB. - The app_mem_req is higher. Thus, 40GB (per node) is requested from the batch scheduler. + - The app_mem_req is 40 (GiB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32 GiB. + The app_mem_req is higher. Thus, 40GiB (per node) is requested from the batch scheduler. Example 2: - - A system with 128 cores per node, 128 GB mem per node is used. + - A system with 128 cores per node, 128 GiB mem per node is used. - The test is launched on 64 cores - - the app_mem_req is 40 (GB) - In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64 GB. - This is higher than the app_mem_req. Thus, 64 GB (per node) is requested from the batch scheduler. + - the app_mem_req is 40 (GiB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64 GiB. + This is higher than the app_mem_req. Thus, 64 GiB (per node) is requested from the batch scheduler. """ # Check that the systems.partitions.extra dict in the ReFrame config contains mem_per_node check_extras_key_defined(test, 'mem_per_node') # Skip if the current partition doesn't have sufficient memory to run the application - msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} GB" + msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} GiB" msg += " memory available (per node) accodring to the current ReFrame configuration," - msg += f" but {app_mem_req} GB is needed" + msg += f" but {app_mem_req} GiB is needed" test.skip_if(test.current_partition.extras['mem_per_node'] < app_mem_req, msg) # Compute what is higher: the requested memory, or the memory available proportional to requested CPUs @@ -421,27 +422,37 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus proportional_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] - # First convert to MB and round - schedulers typically don't allow fractional numbers - # (and we want to reduce roundoff error, hence MB) - # Round up for app_mem_req to be on the save side: - app_mem_req = math.ceil(1024 * app_mem_req) - # Round down for proportional_mem, so we don't ask more than what is available per node - proportional_mem = math.floor(1024 * proportional_mem) - - # Request the maximum of the proportional_mem, and app_mem_req to the scheduler - req_mem_per_node = max(proportional_mem, app_mem_req) if test.current_partition.scheduler.registered_name == 'slurm' or test.current_partition.scheduler.registered_name == 'squeue': # SLURMs --mem defines memory per node, see https://slurm.schedmd.com/sbatch.html + # SLURM uses megabytes and gigabytes, i.e. base-10, so conversion is 1000, not 1024 + # Thus, we convert from GiB (gibibytes) to MB (megabytes) (1024 * 1024 * 1024 / (1000 * 1000) = 1073.741824) + app_mem_req = math.ceil(1073.741824 * app_mem_req) + log(f"Memory requested by application: {app_mem_req} MB") + proportional_mem = math.floor(1073.741824 * proportional_mem) + log(f"Memory proportional to the core count: {proportional_mem} MB") + + # Request the maximum of the proportional_mem, and app_mem_req to the scheduler + req_mem_per_node = max(proportional_mem, app_mem_req) + test.extra_resources = {'memory': {'size': '%sM' % req_mem_per_node }} - log(f"Requested {req_mem_per_node}MB per node from the SLURM batch scheduler") + log(f"Requested {req_mem_per_node} MB per node from the SLURM batch scheduler") + elif test.current_partition.scheduler.registered_name == 'torque': # Torque/moab requires asking for --pmem (--mem only works single node and thus doesnt generalize) # See https://docs.adaptivecomputing.com/10-0-1/Torque/torque.htm#topics/torque/3-jobs/3.1.3-requestingRes.htm - req_mem_per_task = req_mem_per_node / test.num_tasks_per_node + # Units are MiB according to the documentation, thus, we simply multiply with 1024 + # We immediately divide by num_tasks_per_node (before rounding), since -pmem specifies memroy _per process_ + app_mem_req_task = math.ceil(1024 * app_mem_req / test.num_tasks_per_node) + proportional_mem_task = math.floor(1024 * proportional_mem / test.num_tasks_per_node) + + # Request the maximum of the proportional_mem, and app_mem_req to the scheduler + req_mem_per_task = max(proportional_mem_task, app_mem_req_task) + # We assume here the reframe config defines the extra resource memory as asking for pmem # i.e. 'options': ['--pmem={size}'] test.extra_resources = {'memory': {'size': '%smb' % req_mem_per_task }} - log(f"Requested {req_mem_per_task}MB per task from the torque batch scheduler") + log(f"Requested {req_mem_per_task} MiB per task from the torque batch scheduler") + else: logger = rflog.getlogger() msg = "hooks.req_memory_per_node does not support the scheduler you configured" From e2de0ee69dae59531a23d876f319f09cc295d365 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 16:28:40 +0200 Subject: [PATCH 054/119] Make memory requirement tighter, so that we don't exceed the 1 GB/core for large task counts --- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index b98807d5..050e43d3 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -99,7 +99,7 @@ def run_after_setup(self): @run_after('setup') def request_mem(self): - memory_required = self.num_tasks_per_node * 1 + 2 + memory_required = self.num_tasks_per_node * 0.9 + 4 hooks.req_memory_per_node(test=self, app_mem_req=memory_required) @run_after('setup') From 48878d187f4f035fd4fc45b05e92e07ace98d8ca Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 16 May 2024 16:55:37 +0200 Subject: [PATCH 055/119] Add some memory for standard system configs. Will do aws_mc later... --- config/github_actions.py | 6 +++ config/it4i_karolina.py | 5 +++ config/izum_vega.py | 83 +++++++++++++++++++++------------------- config/surf_snellius.py | 12 +++++- 4 files changed, 66 insertions(+), 40 deletions(-) diff --git a/config/github_actions.py b/config/github_actions.py index 5328f6f3..0060b7ab 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -26,6 +26,12 @@ } ], 'max_jobs': 1 + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + # This is a fictional amount, GH actions probably has less, but only does --dry-run + 'mem_per_node': 30 # in GiB + }, } ] } diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py index 90062c85..d395d911 100644 --- a/config/it4i_karolina.py +++ b/config/it4i_karolina.py @@ -53,6 +53,11 @@ 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 219.345 # in GiB + }, 'descr': 'CPU Universal Compute Nodes, see https://docs.it4i.cz/karolina/hardware-overview/' }, # We don't have GPU budget on Karolina at this time diff --git a/config/izum_vega.py b/config/izum_vega.py index 77d75796..3adb1504 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -66,45 +66,50 @@ }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, - { - 'name': 'gpu', - 'scheduler': 'slurm', - 'prepare_cmds': [ - 'source %s' % common_eessi_init(), - # Pass job environment variables like $PATH, etc., into job steps - 'export SLURM_EXPORT_ENV=ALL', - # Needed when using srun launcher - # 'export SLURM_MPI_TYPE=pmix', # WARNING: this broke the GROMACS on Vega - # Avoid https://github.com/EESSI/software-layer/issues/136 - # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1) - 'export OMPI_MCA_pml=ucx', - ], - 'launcher': 'mpirun', - # Use --export=None to avoid that login environment is passed down to submitted jobs - 'access': ['-p gpu', '--export=None'], - 'environs': ['default'], - 'max_jobs': 60, - 'devices': [ - { - 'type': DEVICE_TYPES[GPU], - 'num_devices': 4, - } - ], - 'resources': [ - { - 'name': '_rfm_gpu', - 'options': ['--gpus-per-node={num_gpus_per_node}'], - }, - { - 'name': 'memory', - 'options': ['--mem={size}'], - } - ], - 'features': [ - FEATURES[GPU], - ] + list(SCALES.keys()), - 'descr': 'GPU partition, see https://en-doc.vega.izum.si/architecture/' - }, +# { +# 'name': 'gpu', +# 'scheduler': 'slurm', +# 'prepare_cmds': [ +# 'source %s' % common_eessi_init(), +# # Pass job environment variables like $PATH, etc., into job steps +# 'export SLURM_EXPORT_ENV=ALL', +# # Needed when using srun launcher +# # 'export SLURM_MPI_TYPE=pmix', # WARNING: this broke the GROMACS on Vega +# # Avoid https://github.com/EESSI/software-layer/issues/136 +# # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1) +# 'export OMPI_MCA_pml=ucx', +# ], +# 'launcher': 'mpirun', +# # Use --export=None to avoid that login environment is passed down to submitted jobs +# 'access': ['-p gpu', '--export=None'], +# 'environs': ['default'], +# 'max_jobs': 60, +# 'devices': [ +# { +# 'type': DEVICE_TYPES[GPU], +# 'num_devices': 4, +# } +# ], +# 'resources': [ +# { +# 'name': '_rfm_gpu', +# 'options': ['--gpus-per-node={num_gpus_per_node}'], +# }, +# { +# 'name': 'memory', +# 'options': ['--mem={size}'], +# } +# ], +# 'features': [ +# FEATURES[GPU], +# ] + list(SCALES.keys()), +# 'extras': { +# # Make sure to round down, otherwise a job might ask for more mem than is available +# # per node +# 'mem_per_node': 476.837 # in GiB (should be checked, its unclear from slurm.conf) +# }, +# 'descr': 'GPU partition, see https://en-doc.vega.izum.si/architecture/' +# }, ] }, ], diff --git a/config/surf_snellius.py b/config/surf_snellius.py index 542f3ee2..d8bcc36c 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -53,6 +53,11 @@ 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 213.623 # in GiB + }, 'descr': 'AMD Rome CPU partition with native EESSI stack' }, { @@ -73,7 +78,9 @@ FEATURES[CPU], ] + list(SCALES.keys()), 'extras': { - 'mem_per_node': 336 + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 320.434 # in GiB }, 'descr': 'AMD Genoa CPU partition with native EESSI stack' }, @@ -108,6 +115,9 @@ ] + valid_scales_snellius_gpu, 'extras': { GPU_VENDOR: GPU_VENDORS[NVIDIA], + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 457.763 # in GiB }, 'descr': 'Nvidia A100 GPU partition with native EESSI stack' }, From 94e616197a89e6dd9ccbf073fc3be9c7970ae8f9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Thu, 23 May 2024 14:03:10 +0200 Subject: [PATCH 056/119] Apply suggestions from code review Fix CI issues Co-authored-by: Davide Grassano <34096612+Crivella@users.noreply.github.com> --- config/github_actions.py | 2 +- config/izum_vega.py | 88 ++++++++++++++++++++-------------------- eessi/testsuite/hooks.py | 11 +++-- 3 files changed, 50 insertions(+), 51 deletions(-) diff --git a/config/github_actions.py b/config/github_actions.py index 0060b7ab..b6555d28 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -25,7 +25,7 @@ 'options': ['--mem={size}'], } ], - 'max_jobs': 1 + 'max_jobs': 1, 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node diff --git a/config/izum_vega.py b/config/izum_vega.py index 3adb1504..765d6a69 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -66,50 +66,50 @@ }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, -# { -# 'name': 'gpu', -# 'scheduler': 'slurm', -# 'prepare_cmds': [ -# 'source %s' % common_eessi_init(), -# # Pass job environment variables like $PATH, etc., into job steps -# 'export SLURM_EXPORT_ENV=ALL', -# # Needed when using srun launcher -# # 'export SLURM_MPI_TYPE=pmix', # WARNING: this broke the GROMACS on Vega -# # Avoid https://github.com/EESSI/software-layer/issues/136 -# # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1) -# 'export OMPI_MCA_pml=ucx', -# ], -# 'launcher': 'mpirun', -# # Use --export=None to avoid that login environment is passed down to submitted jobs -# 'access': ['-p gpu', '--export=None'], -# 'environs': ['default'], -# 'max_jobs': 60, -# 'devices': [ -# { -# 'type': DEVICE_TYPES[GPU], -# 'num_devices': 4, -# } -# ], -# 'resources': [ -# { -# 'name': '_rfm_gpu', -# 'options': ['--gpus-per-node={num_gpus_per_node}'], -# }, -# { -# 'name': 'memory', -# 'options': ['--mem={size}'], -# } -# ], -# 'features': [ -# FEATURES[GPU], -# ] + list(SCALES.keys()), -# 'extras': { -# # Make sure to round down, otherwise a job might ask for more mem than is available -# # per node -# 'mem_per_node': 476.837 # in GiB (should be checked, its unclear from slurm.conf) -# }, -# 'descr': 'GPU partition, see https://en-doc.vega.izum.si/architecture/' -# }, +{ + # 'name': 'gpu', + # 'scheduler': 'slurm', + # 'prepare_cmds': [ + # 'source %s' % common_eessi_init(), + # # Pass job environment variables like $PATH, etc., into job steps + # 'export SLURM_EXPORT_ENV=ALL', + # # Needed when using srun launcher + # # 'export SLURM_MPI_TYPE=pmix', # WARNING: this broke the GROMACS on Vega + # # Avoid https://github.com/EESSI/software-layer/issues/136 + # # Can be taken out once we don't care about old OpenMPI versions anymore (pre-4.1.1) + # 'export OMPI_MCA_pml=ucx', + # ], + # 'launcher': 'mpirun', + # # Use --export=None to avoid that login environment is passed down to submitted jobs + # 'access': ['-p gpu', '--export=None'], + # 'environs': ['default'], + # 'max_jobs': 60, + # 'devices': [ + # { + # 'type': DEVICE_TYPES[GPU], + # 'num_devices': 4, + # } + # ], + # 'resources': [ + # { + # 'name': '_rfm_gpu', + # 'options': ['--gpus-per-node={num_gpus_per_node}'], + # }, + # { + # 'name': 'memory', + # 'options': ['--mem={size}'], + # } + # ], + # 'features': [ + # FEATURES[GPU], + # ] + list(SCALES.keys()), + # 'extras': { + # # Make sure to round down, otherwise a job might ask for more mem than is available + # # per node + # 'mem_per_node': 476.837 # in GiB (should be checked, its unclear from slurm.conf) + # }, + # 'descr': 'GPU partition, see https://en-doc.vega.izum.si/architecture/' + # }, ] }, ], diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index a97e7d45..9e838904 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -409,7 +409,6 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): """ # Check that the systems.partitions.extra dict in the ReFrame config contains mem_per_node check_extras_key_defined(test, 'mem_per_node') - # Skip if the current partition doesn't have sufficient memory to run the application msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} GiB" msg += " memory available (per node) accodring to the current ReFrame configuration," @@ -422,7 +421,8 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus proportional_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] - if test.current_partition.scheduler.registered_name == 'slurm' or test.current_partition.scheduler.registered_name == 'squeue': + scheduler_name = test.current_partition.scheduler.registered_name + if scheduler_name == 'slurm' or scheduler_name == 'squeue': # SLURMs --mem defines memory per node, see https://slurm.schedmd.com/sbatch.html # SLURM uses megabytes and gigabytes, i.e. base-10, so conversion is 1000, not 1024 # Thus, we convert from GiB (gibibytes) to MB (megabytes) (1024 * 1024 * 1024 / (1000 * 1000) = 1073.741824) @@ -434,10 +434,10 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): # Request the maximum of the proportional_mem, and app_mem_req to the scheduler req_mem_per_node = max(proportional_mem, app_mem_req) - test.extra_resources = {'memory': {'size': '%sM' % req_mem_per_node }} + test.extra_resources = {'memory': {'size': f'{req_mem_per_node}M' }} log(f"Requested {req_mem_per_node} MB per node from the SLURM batch scheduler") - elif test.current_partition.scheduler.registered_name == 'torque': + elif scheduler_name == 'torque': # Torque/moab requires asking for --pmem (--mem only works single node and thus doesnt generalize) # See https://docs.adaptivecomputing.com/10-0-1/Torque/torque.htm#topics/torque/3-jobs/3.1.3-requestingRes.htm # Units are MiB according to the documentation, thus, we simply multiply with 1024 @@ -450,7 +450,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): # We assume here the reframe config defines the extra resource memory as asking for pmem # i.e. 'options': ['--pmem={size}'] - test.extra_resources = {'memory': {'size': '%smb' % req_mem_per_task }} + test.extra_resources = {'memory': {'size': f'{req_mem_per_task}mb'}} log(f"Requested {req_mem_per_task} MiB per task from the torque batch scheduler") else: @@ -462,7 +462,6 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): msg += " Please expand the functionality of hooks.req_memory_per_node for your scheduler." # Warnings will, at default loglevel, be printed on stdout when executing the ReFrame command logger.warning(msg) - def set_modules(test: rfm.RegressionTest): """ From 360df652893fa8913a6e7fcad860f029355e6b91 Mon Sep 17 00:00:00 2001 From: Davide Grassano <34096612+Crivella@users.noreply.github.com> Date: Thu, 23 May 2024 14:15:47 +0200 Subject: [PATCH 057/119] Apply suggestions from code review Apply fixes --- config/izum_vega.py | 2 +- eessi/testsuite/hooks.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/config/izum_vega.py b/config/izum_vega.py index 765d6a69..f7193aed 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -66,7 +66,7 @@ }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, -{ + # { # 'name': 'gpu', # 'scheduler': 'slurm', # 'prepare_cmds': [ diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 9e838904..72a72569 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -373,11 +373,6 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic log(f'valid_systems set to {test.valid_systems}') -## TODO: function should take everything in MB, as schedulers (at least slurm) does not except asking for fractional memory -## ie --mem=7.0G is invalid. This should be done as --mem=7168M. -## It's probably better if this function does everything in MB, and the ReFrame config also specifies available mem per node in MB. -## Then, we should make sure the numbers are integers by rounding up for app_mem_req (1 MB more should never really be an issue) -## and probably down for the proportional_mem (as to not ask for more than the equivalent share of a core) def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): """ This hook will request a specific amount of memory per node to the batch scheduler. @@ -434,7 +429,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): # Request the maximum of the proportional_mem, and app_mem_req to the scheduler req_mem_per_node = max(proportional_mem, app_mem_req) - test.extra_resources = {'memory': {'size': f'{req_mem_per_node}M' }} + test.extra_resources = {'memory': {'size': f'{req_mem_per_node}M'}} log(f"Requested {req_mem_per_node} MB per node from the SLURM batch scheduler") elif scheduler_name == 'torque': @@ -463,6 +458,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): # Warnings will, at default loglevel, be printed on stdout when executing the ReFrame command logger.warning(msg) + def set_modules(test: rfm.RegressionTest): """ Skip current test if module_name is not among a list of modules, From b66c62722133ce561196b0b201a0c5436702a89d Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 23 May 2024 14:21:28 +0200 Subject: [PATCH 058/119] Linting --- eessi/testsuite/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index ac408b45..ee679295 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -150,7 +150,7 @@ def check_proc_attribute_defined(test: rfm.RegressionTest, attribute) -> bool: ) raise AttributeError(msg) - + def check_extras_key_defined(test: rfm.RegressionTest, extra_key) -> bool: """ Checks if a specific key is defined in the 'extras' dictionary for the current partition From 387bd80891b832013ce17e26476953f031c5e2e3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 23 May 2024 14:46:37 +0200 Subject: [PATCH 059/119] use software.eessi.io repo in CI --- .github/workflows/test.yml | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5686848..73471776 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests for EESSI test suite, using EESSI pilot repo +name: Tests for EESSI test suite, using EESSI production repo on: [push, pull_request, workflow_dispatch] permissions: read-all jobs: @@ -9,39 +9,23 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - "2021.12" + - '2023.06' steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false - - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + - name: Mount EESSI CernVM-FS production repository + uses: eessi/github-action-eessi@e1f8f20638ea417a18d23ab29443ee34794ff900 # v3.1.0 with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: pilot.eessi-hpc.org + eessi_stack_version: ${{matrix.EESSI_VERSION}} - name: Run test suite run: | - source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash - # install latest version of EasyBuild, to install ReFrame with it, - # since that includes the ReFrame test library (hpctestlib) that we rely on - python3 -m venv venv - source venv/bin/activate - pip3 install easybuild - eb --version - export EASYBUILD_PREFIX=$HOME/easybuild - # need to force module generation with --module-only --force because 'pip check' fails - # in EESSI pilot 2021.12, see https://github.com/EESSI/compatibility-layer/issues/152 - eb ReFrame-4.3.3.eb || eb ReFrame-4.3.3.eb --module-only --force - - # load ReFrame - module use $HOME/easybuild/modules/all - - module load ReFrame/4.3.3 + module load ReFrame reframe --version # configure ReFrame (cfr. https://reframe-hpc.readthedocs.io/en/stable/manpage.html#environment) From 471becc9c14d7ccc367feb4de4f8443ac10fa6f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 23 May 2024 15:12:34 +0200 Subject: [PATCH 060/119] bump version to 0.2.0 --- RELEASE_NOTES | 2 +- pyproject.toml | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 3efe3cc4..79bd3dab 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,7 +1,7 @@ This file contains a description of the major changes to the EESSI test suite. For more detailed information, please see the git log. -v0.2.0 (7 march 2024) +v0.2.0 (7 March 2024) --------------------- This is a minor release of the EESSI test-suite diff --git a/pyproject.toml b/pyproject.toml index 3c374a5c..2b3b607c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "eessi-testsuite" -version = "0.1.0" +version = "0.2.0" description = "Test suite for the EESSI software stack" readme = "README.md" license = {file = "LICENSE"} diff --git a/setup.cfg b/setup.cfg index 87b688e7..49a7b178 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eessi-testsuite -version = 0.1.0 +version = 0.2.0 description = Test suite for the EESSI software stack long_description = file: README.md long_description_content_type = text/markdown From 8fcdb704d9a065f5a22d869312345bc10d1cdafc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 23 May 2024 15:32:21 +0200 Subject: [PATCH 061/119] add notes on release management to README --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 5049bbf8..72878010 100644 --- a/README.md +++ b/README.md @@ -98,3 +98,17 @@ is that it is easy to pull in updates from a feature branch using `git pull`. You can also push back changes to the feature branch directly, but note that you are pushing to the Github fork of another Github user, so _make sure they are ok with that_ before doing so! + +## Release management + +When a release of the EESSI test suite is made, the following things must be taken care of: + +- Version bump: in both `pyproject.toml` and `setup.cfg`; +- Release notes: in `RELEASE_NOTES` + in GitHub release (cfr. https://github.com/EESSI/test-suite/releases/tag/v0.2.0); +- Tag release on GitHub + publish release (incl. release notes); +- Publishing release to PyPI: + ``` + # example for version 0.2.0 + python setup.py sdist + twine upload dist/eessi_testsuite-0.2.0.tar.gz + ``` From e4d8a5183e9246e597c38ddd12f2be11940d86e5 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Fri, 24 May 2024 16:26:51 +0200 Subject: [PATCH 062/119] update hortense config --- config/vsc_hortense.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index fbfa9e4c..f8a83b4f 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -54,6 +54,11 @@ def command(self, job): 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) + }, }, { 'name': 'cpu_rome_512gb', @@ -81,6 +86,11 @@ def command(self, job): 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 511.983 # in GiB + }, }, { 'name': 'cpu_milan', @@ -108,6 +118,11 @@ def command(self, job): 'features': [ FEATURES[CPU], ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) + }, }, { 'name': 'gpu_rome_a100_40gb', @@ -131,6 +146,9 @@ def command(self, job): ] + list(SCALES.keys()), 'extras': { GPU_VENDOR: GPU_VENDORS[NVIDIA], + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 256.000 # in GiB }, 'resources': [ { @@ -172,6 +190,9 @@ def command(self, job): ] + list(SCALES.keys()), 'extras': { GPU_VENDOR: GPU_VENDORS[NVIDIA], + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 511.983 # in GiB }, 'resources': [ { From 58a716d1225384ad678514ecb53fe4e577dd278b Mon Sep 17 00:00:00 2001 From: crivella Date: Fri, 24 May 2024 17:31:28 +0200 Subject: [PATCH 063/119] lint --- config/vsc_hortense.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index f8a83b4f..f349bf60 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -6,7 +6,9 @@ from reframe.core.backends import register_launcher from reframe.core.launchers import JobLauncher -from eessi.testsuite.common_config import common_logging_config, common_general_config, common_eessi_init +from eessi.testsuite.common_config import (common_eessi_init, + common_general_config, + common_logging_config) from eessi.testsuite.constants import * # noqa: F403 account = "my-slurm-account" @@ -56,8 +58,8 @@ def command(self, job): ] + list(SCALES.keys()), 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available - # per node - 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) + # per node + 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) }, }, { From 6e429af96f2297f5f881ca178360350a99e4a9f8 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 29 May 2024 19:05:35 +0200 Subject: [PATCH 064/119] 1. Scaled memory with the number of tasks per node. 2. Increased time limit to account for tuning that takes longer on large number of cores. --- .../testsuite/tests/apps/espresso/espresso.py | 15 ++++++------ .../tests/apps/espresso/src/madelung.py | 23 +++++++------------ 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 37f81344..98b0017e 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -26,7 +26,7 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): scale = parameter(SCALES.keys()) valid_prog_environs = ['default'] valid_systems = ['*'] - time_limit = '30m' + time_limit = '180m' # Need to check if QuantumESPRESSO also gets listed. module_name = parameter(find_modules('ESPResSo')) # device type is parameterized for an impending CUDA ESPResSo module. @@ -66,12 +66,6 @@ def set_tag_ci(self): if (self.benchmark_info[0] == 'mpi.ionic_crystals.p3m'): self.tags.add('ionic_crystals_p3m') - - @run_after('init') - def set_mem(self): - """ Setting an extra job option of memory. """ - self.extra_resources = {'memory': {'size': '50GB'}} - @run_after('init') def set_executable_opts(self): """Set executable opts based on device_type parameter""" @@ -89,6 +83,13 @@ def set_num_tasks_per_node(self): for 1 node and 2 node options where the request is for full nodes.""" hooks.assign_tasks_per_compute_unit(self, COMPUTE_UNIT[CPU]) + @run_after('setup') + def set_mem(self): + """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at + least 1 GB per core of memory.""" + mem_required_per_node = str(self.num_tasks_per_node * 1) + 'GB' + self.extra_resources = {'memory': {'size': mem_required_per_node}} + @deferrable def assert_completion(self): '''Check completion''' diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 628d8eab..2cf6fea0 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -105,6 +105,7 @@ def get_normalized_values_per_ion(system): rtol_forces = 0. rtol_abs_forces = 0. # run checks +print("Executing sanity checks...\n") forces = np.copy(system.part.all().f) energy, p_scalar, p_tensor = get_normalized_values_per_ion(system) ref_energy, ref_pressure = get_reference_values_per_ion(base_vector) @@ -115,22 +116,12 @@ def get_normalized_values_per_ion(system): np.testing.assert_allclose(forces, 0., atol=atol_forces, rtol=rtol_forces) np.testing.assert_allclose(np.median(np.abs(forces)), 0., atol=atol_abs_forces, rtol=rtol_abs_forces) - -print("Executing sanity checks...\n") -if (np.all([np.allclose(energy, ref_energy, atol=atol_energy, rtol=rtol_energy), - np.allclose(p_scalar, np.trace(ref_pressure) / 3., - atol=atol_pressure, rtol=rtol_pressure), - np.allclose(p_tensor, ref_pressure, atol=atol_pressure, rtol=rtol_pressure), - np.allclose(forces, 0., atol=atol_forces, rtol=rtol_forces), - np.allclose(np.median(np.abs(forces)), 0., atol=atol_abs_forces, rtol=rtol_abs_forces)])): - print("Final convergence met with tolerances: \n\ +print("Final convergence met with tolerances: \n\ energy: ", atol_energy, "\n\ p_scalar: ", atol_pressure, "\n\ p_tensor: ", atol_pressure, "\n\ forces: ", atol_forces, "\n\ abs_forces: ", atol_abs_forces, "\n") -else: - print("At least one parameter did not meet the tolerance, see the log above.\n") print("Sampling runtime...\n") # sample runtime @@ -142,11 +133,13 @@ def get_normalized_values_per_ion(system): tock = time.time() timings.append((tock - tick) / n_steps) +print("10 steps executed...\n") # write results to file header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std"\n' report = f'"{"weak scaling" if args.weak_scaling else "strong scaling"}",{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},{np.mean(timings):.3e},{np.std(timings, ddof=1):.3e}\n' print(report) -if pathlib.Path(args.output).is_file(): - header = "" -with open(args.output, "a") as f: - f.write(header + report) + +# if pathlib.Path(args.output).is_file(): +# header = "" +# with open(args.output, "a") as f: +# f.write(header + report) From e4321a8cc4088634983a853539fd91d978f9f44c Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 29 May 2024 19:08:43 +0200 Subject: [PATCH 065/119] 1. Increased time limit again to 5 hours for 16 node tests. This is a temporary fix until the mesh size can be fixed based on extrapolation. --- eessi/testsuite/tests/apps/espresso/espresso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 98b0017e..d39c4aaa 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -26,7 +26,7 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): scale = parameter(SCALES.keys()) valid_prog_environs = ['default'] valid_systems = ['*'] - time_limit = '180m' + time_limit = '300m' # Need to check if QuantumESPRESSO also gets listed. module_name = parameter(find_modules('ESPResSo')) # device type is parameterized for an impending CUDA ESPResSo module. From c06e32f2d0a692144954b7338b3183253c484b23 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 29 May 2024 19:18:31 +0200 Subject: [PATCH 066/119] Deleting the tar file. --- .../apps/espresso/src/scripts_Espresso.tar.gz | Bin 3089 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 eessi/testsuite/tests/apps/espresso/src/scripts_Espresso.tar.gz diff --git a/eessi/testsuite/tests/apps/espresso/src/scripts_Espresso.tar.gz b/eessi/testsuite/tests/apps/espresso/src/scripts_Espresso.tar.gz deleted file mode 100644 index 24e2621fec80e082c1830617209e16fa63c8df4e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3089 zcmV+s4DRzEiwFP!000001MM1XZ`(MspYRr0(9qv~LhNs*Q44Y*X-3GCt9eGlNARK` z{T}LkZ+NObC9il|WYK&fiSf!LuI+ez*LHn!zF_3_*~uvrXPl%NzhZ&Zkl=g~2{Maf zM&O##OcFk8FfE1B&>?2V7!iCXZ)wK%Nx^dx(1c_xjD*ahX)b}0Bs2*JJR?gUMzaD~ zz)+rqEF%*1kStpY6oU_Mjz5w&EMXaq$w@wqqkz1N0+t9yC@6#@V!=W(MK(xqh#H(# z8jwQ{e5s6h(jzQ__ZhiinSf{F)gmf>J;F2KVNi)`XN0FnW`eE-iK$#sZq9l&xsXJO zV!Gfd%wz$~U~acj9Fr*{xnQ$A?g2I6k{^%G-+uUbP7aPgksl9EPY;gIKm8qW<$}Xo zcEd_e(K3xAU<~ugXd(+x8yLKQefsJxP#nBGdUtgG33MliN9V_{&(6rh_82K0~Eb*4N6#tWIGQ7p>&egant&@2wgg5EH2X~3cz z(2J4)g6hlu0v`nkTu%v-uz{ zd5FP3Q5y|x!XXryNHZn`Uxu3_R^_cNZ&(b`opHzpG73bEK(jf-H|SYCr&KQDXj(tY zXxY3<@+Hg|;x9=xO%Y{e+%T{XvU$E_i9AtnjF5>Si&Qj?^_MIO7E794lN0Yf5z#V_ zRd@XSQLGj?3JXn+gjcx`DOkvXCbj%4+5HmrJ zaVh9q87k002Z##EiQzs!Gym(B!A|=9TSl+@BA{^u;h`d<;(z=oKfnX?$*5c()8d|H z0a!--@UU6uX}&UKvb#AIvE3HrK^&+f5+&sXiRQ-cerz}=R#bBSTGcaK`^Ni;y ztk>48mbrw`jEf1o3*uZrUYjT^qr%>pdOKq8Pj(`+{DB#liv9@1qz?B+KVrPW24yCA z@=QR*x!s2dPGBT~0>a^a3So^g-OOP#Q9xEp=D%C7Gey2lvx8+w7x9K#l1o_7s2u8q zLAf%kx}bIfjHrP-$ehVZ#%3%7zHD+s21ON% z9E#u;B;eVWH$Vw8qX9Eo!?iS-XGu$FZ;kuPFbU6=VD_I`c;7Oj5Rxq`qcobAqZWk} zRftGfdchYPNNv*^-qQ;aD5mgwQw8uacvs!Ho16i8lUn_1m zJcgJx)=cDDVAg7S19IFe2O9OC1ATkAQm`7mb&1(xgDKJYiQMC)OAu}5gUC=0nMuTRWS~SCULs+$lGCCgP4b0m2Yt*QI8bbgR z>_iAO$I#T}Q|R5GIf3fVwDyV`W4vQCS{N4&8czKK(^be5Y#0E`1Mgh8@S|km;ANvX z=Fdr$XiDAh5rc@TwiibGRc|$m##hRjkJ7ZWi-!H}753@SPS4pI=7oJ(Ei^>BegMg- zk!r;F0YtZrh%i|N04@wn;6}>DxUKocRgYZvNPM}j_V&uxhR&T)*9M1vrQTw07B)*q@sH#VDkYe}y#`Si9l{MIr`@SRDzhR@Q`k?OBgT^M*{9^>@A?voz(pUAAcdMNXwNdju;&O}$eQ+t#C`xQJN@i68Ii zR8+i7v&9YM4_n-DF;d(ZU_7ZEW}=D?Mw4S}Fam<`;o?%It;s6UICd4*5TVzAfz+yM zRb{mbLMR{%_SoQh3q%+d`w~U+t(28ii>4_{Lc>sKeI*B-*RTNPZ(#-FQGkHtC2(hl z&9}B-G*rvwnZAT%t@m_Q&fvOCBkQiZ7S;7=^6J4zJvih%j@0Y5o{t9<>34wgS_UNoXy7+j%nL!NnH2Ar)l(}zt ztQ-cd>rvRmgnTx3%C|U`XjUI8SSN5blpP)ubH@1i`LFFbzIX5Z*Y#Y#JOBM2@P95wZGG|e z{GJPcPH#AZ?CM&c51G<-dV7|=hj*TKe+@PiJ9{2d_@kks7(shIDk;__IKzs>-5vUS zNa5_V7q$%)JG)5hZpvtAln~F_+wl;w=UAhWvrPzK`HEy`#~lH&{_0U;{1)7)PK9~9$*c3b`-v6`JTT`snSkP&+~VNpa}dbOZnDtceLHM zHAOY;kylbUmOa{gz*5zpq8M79=T#KtY@AJsq2DY26bV?=vDGw&mgBi<+JAJ2ckl?L zF_+)++aJOg^1thO8{hwI*Xi>A_kb-QFPcvsNeBb@L9O|cvZN9N2Ah?%z&rIimRsIq zX9yh!ykFw2riC*+2MnzYT9FC#8pi!&T(d+8XRx(`7P{Q87}H{WQ57jtS1D4LDN@%d zQWYxP%hpG^!02<%vci}tdffRvThCt4K&Cc}=BQXKEx5yzFSy`FeQe+cVQ88fy~9p3 znp+~D;`9V;-sl|@+FeK`#f=t9yr{Ihv?QM>dqV46tYfIkxrRXRfEDP%3`22)=CeoY zJJ=+M7<#<116zN<5weKcSnu~?!U2zAbF9xZhSQJfl*MEHe8E5mus4mt7kAMzU+PK* zs$B}`k)q-bZF^CU=)Aw;@t#rKC*kq2D2WB^BV{C^550F9yn217hzb2+{bdxR*N z^@!6r-2SsTy!~0?cewlYY!(F(tn=b49A3Z#ghEj#Ig=S4en`nOL!}Vwgu&K1k%!H& zV?G!Pe{^2`tH2eXg2}$Ej4#xGd+6M+|2DMpUH$(a@brhlG)e}r3K!bbvzG_wuilb= z9}j-Uw!QD$`>@qnjhCkk3G$bOW!S$f}{7U9Q zFmPaVXxSvlLt5YCVFpz3fp0n1ko>fE7&x2B+kppYj!mYV3psGCJtRsCx!?)B0w#_o zcn&FoSk|CE0`B{kH@IQRjhZHY`n;i%X#$UG`~4uz`!F7k%TFsMGq3r$2Tcws|EC2R z-uoX3zn#N~-2WD Date: Tue, 4 Jun 2024 12:50:55 +0200 Subject: [PATCH 067/119] Increasing the pressure tolerance for higher node counts as instructed by Jean-Noel. --- eessi/testsuite/tests/apps/espresso/src/madelung.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 2cf6fea0..0c848dfc 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -101,7 +101,8 @@ def get_normalized_values_per_ion(system): atol_abs_forces = 2e-6 rtol_energy = 5e-6 -rtol_pressure = 2e-5 +#rtol_pressure = 2e-5 +rtol_pressure = 1e-4 rtol_forces = 0. rtol_abs_forces = 0. # run checks From eea35380407b6b8dfce398831881ab79b7387483 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 4 Jun 2024 14:29:30 +0200 Subject: [PATCH 068/119] Introduced a performance function for weak scaling which is the mean time per step. --- eessi/testsuite/tests/apps/espresso/espresso.py | 4 ++++ eessi/testsuite/tests/apps/espresso/src/madelung.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index d39c4aaa..aee07353 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -111,3 +111,7 @@ def assert_sanity(self): self.assert_convergence(), ]) + @performance_function('s/step') + def perf(self): + return sn.extractsingle(r'^Performance:\s+(?P\S+)', self.stdout, 'perf', float) + diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 0c848dfc..7d55bd0d 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -138,8 +138,11 @@ def get_normalized_values_per_ion(system): # write results to file header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std"\n' report = f'"{"weak scaling" if args.weak_scaling else "strong scaling"}",{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},{np.mean(timings):.3e},{np.std(timings, ddof=1):.3e}\n' +print(header) print(report) +print(f"Performance: {np.mean(timings):.3e} \n") + # if pathlib.Path(args.output).is_file(): # header = "" # with open(args.output, "a") as f: From a2c660faa87d0689360d815b2c9a92eb8ed10d46 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 4 Jun 2024 14:57:32 +0200 Subject: [PATCH 069/119] Trying to make the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 15 ++++++--------- .../testsuite/tests/apps/espresso/src/madelung.py | 14 +++++++------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index aee07353..9a3a8ecb 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -1,11 +1,10 @@ """ -This module tests Espresso in available modules containing substring 'ESPResSo' which is different from Quantum Espresso. -Tests included: +This module tests Espresso in available modules containing substring 'ESPResSo' which is different from Quantum +Espresso. Tests included: - P3M benchmark - Ionic crystals - Weak scaling - - Strong scaling -Weak and strong scaling are options that are needed to be provided tothe script and the system is either scaled based on -number of cores or kept constant. + - Strong scaling Weak and strong scaling are options that are needed to be provided to the script and the system is + either scaled based on number of cores or kept constant. """ import reframe as rfm @@ -14,15 +13,14 @@ from reframe.core.builtins import parameter, run_after # added only to make the linter happy from reframe.utility import reframe -from hpctestlib.microbenchmarks.mpi.osu import osu_benchmark - from eessi.testsuite import hooks, utils from eessi.testsuite.constants import * from eessi.testsuite.utils import find_modules, log @rfm.simple_test class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): - '''''' + + scale = parameter(SCALES.keys()) valid_prog_environs = ['default'] valid_systems = ['*'] @@ -45,7 +43,6 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): @run_after('init') def run_after_init(self): """hooks to run after init phase""" - # Filter on which scales are supported by the partitions defined in the ReFrame configuration hooks.filter_supported_scales(self) diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 7d55bd0d..ce41d61a 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -21,7 +21,6 @@ import espressomd.version import espressomd.electrostatics import argparse -import pathlib import time import numpy as np @@ -45,6 +44,7 @@ help="Strong scaling benchmark (Amdahl's law: constant total work)") args = parser.parse_args() + def get_reference_values_per_ion(base_vector): madelung_constant = -1.74756459463318219 base_tensor = base_vector * np.eye(3) @@ -52,6 +52,7 @@ def get_reference_values_per_ion(base_vector): ref_pressure = madelung_constant * base_tensor / np.trace(base_tensor) return ref_energy, ref_pressure + def get_normalized_values_per_ion(system): energy = system.analysis.energy()["coulomb"] p_scalar = system.analysis.pressure()["coulomb"] @@ -60,6 +61,7 @@ def get_normalized_values_per_ion(system): V = system.volume() return 2. * energy / N, 2. * p_scalar * V / N, 2. * p_tensor * V / N + # initialize system system = espressomd.System(box_l=[100., 100., 100.]) system.time_step = 0.01 @@ -96,12 +98,15 @@ def get_normalized_values_per_ion(system): print("Algorithm executed. \n") +# Old rtol_pressure = 2e-5 +# This resulted in failures especially at high number of nodes therefore increased +# to a larger value. + atol_energy = atol_pressure = 1e-12 atol_forces = 1e-5 atol_abs_forces = 2e-6 rtol_energy = 5e-6 -#rtol_pressure = 2e-5 rtol_pressure = 1e-4 rtol_forces = 0. rtol_abs_forces = 0. @@ -142,8 +147,3 @@ def get_normalized_values_per_ion(system): print(report) print(f"Performance: {np.mean(timings):.3e} \n") - -# if pathlib.Path(args.output).is_file(): -# header = "" -# with open(args.output, "a") as f: -# f.write(header + report) From cbe7fe9865bdeaa142a7e4cc37d38d4dacf3ab45 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 4 Jun 2024 15:00:39 +0200 Subject: [PATCH 070/119] Linter changes. --- eessi/testsuite/tests/apps/espresso/espresso.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 9a3a8ecb..9fbccf58 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -17,10 +17,10 @@ from eessi.testsuite.constants import * from eessi.testsuite.utils import find_modules, log + @rfm.simple_test class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): - scale = parameter(SCALES.keys()) valid_prog_environs = ['default'] valid_systems = ['*'] @@ -39,7 +39,6 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): ('mpi.ionic_crystals.p3m', 'p3m'), ], fmt=lambda x: x[0], loggable=True) - @run_after('init') def run_after_init(self): """hooks to run after init phase""" @@ -111,4 +110,3 @@ def assert_sanity(self): @performance_function('s/step') def perf(self): return sn.extractsingle(r'^Performance:\s+(?P\S+)', self.stdout, 'perf', float) - From f434c48a68512d9a8f3fcc5ff786d21f76957c83 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 5 Jun 2024 00:55:03 +0200 Subject: [PATCH 071/119] Removed plot.py as it served no purpose and improved formatting in madelung.py for the linter. --- .../tests/apps/espresso/src/madelung.py | 4 +- .../testsuite/tests/apps/espresso/src/plot.py | 39 ------------------- 2 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 eessi/testsuite/tests/apps/espresso/src/plot.py diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index ce41d61a..1c019e29 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -142,7 +142,9 @@ def get_normalized_values_per_ion(system): print("10 steps executed...\n") # write results to file header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std"\n' -report = f'"{"weak scaling" if args.weak_scaling else "strong scaling"}",{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},{np.mean(timings):.3e},{np.std(timings, ddof=1):.3e}\n' +report = f'''"{"weak scaling" if args.weak_scaling else "strong scaling"}",\ +{n_cores},{node_grid[0]},{node_grid[1]},{node_grid[2]},{len(system.part)},\ +{np.mean(timings):.3e},{np.std(timings,ddof=1):.3e}\n''' print(header) print(report) diff --git a/eessi/testsuite/tests/apps/espresso/src/plot.py b/eessi/testsuite/tests/apps/espresso/src/plot.py deleted file mode 100644 index c9a023c4..00000000 --- a/eessi/testsuite/tests/apps/espresso/src/plot.py +++ /dev/null @@ -1,39 +0,0 @@ -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib.ticker as mtick - -df = pd.read_csv("benchmarks.csv") -df = df.sort_values(by=["mode", "cores", "mpi.x", "mpi.y", "mpi.z"]) - -group = df.query(f"mode == 'strong scaling'") - -fig = plt.figure(figsize=(12, 6)) -ax = fig.subplots().axes -xdata = group["cores"].to_numpy() -ydata = group["mean"].to_numpy() -ax.axline((xdata[0], xdata[0]), slope=1, linestyle="--", color="grey", label="Theoretical maximum") -ax.plot(xdata, ydata[0] / ydata, "o-", label="Measurements") -ax.set_title("Strong scaling") -ax.set_xlabel("Number of cores") -ax.set_ylabel("Speed-up") -ax.set_xscale("log", base=2) -ax.set_yscale("log", base=10) -ax.legend() -plt.show() - -group = df.query(f"mode == 'weak scaling'") - -fig = plt.figure(figsize=(12, 6)) -ax = fig.subplots().axes -xdata = group["cores"].to_numpy() -ydata = group["mean"].to_numpy() -ax.axline((-np.inf, 1), slope=0, linestyle="--", color="grey", label="Theoretical maximum") -ax.plot(xdata, ydata[0] / ydata, "o-", label="Measurements") -ax.set_title("Weak scaling") -ax.set_xlabel("Number of cores") -ax.set_ylabel("Efficiency") -ax.set_xscale("log", base=2) -ax.yaxis.set_major_formatter(mtick.PercentFormatter(1)) -ax.legend() -plt.show() From bd04f8340bacb434cf2db908d17091e1c74f93d8 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 5 Jun 2024 01:02:14 +0200 Subject: [PATCH 072/119] Making linter happy again. --- eessi/testsuite/tests/apps/espresso/src/madelung.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 1c019e29..37d0b44a 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -79,11 +79,11 @@ def get_normalized_values_per_ion(system): if args.weak_scaling: lattice_size = np.multiply(lattice_size, node_grid) system.box_l = np.multiply(lattice_size, base_vector) -for j in range(lattice_size[0]): - for k in range(lattice_size[1]): - for l in range(lattice_size[2]): - _ = system.part.add(pos=np.multiply([j, k, l], base_vector), - q=(-1.)**(j + k + l), fix=3 * [True]) +for var_j in range(lattice_size[0]): + for var_k in range(lattice_size[1]): + for var_l in range(lattice_size[2]): + _ = system.part.add(pos=np.multiply([var_j, var_k, var_l], base_vector), + q=(-1.)**(var_j + var_k + var_l), fix=3 * [True]) # setup P3M algorithm algorithm = espressomd.electrostatics.P3M From ef21ed5259c787b3993732df78e9d4c81b042f56 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 6 Jun 2024 23:59:19 +0200 Subject: [PATCH 073/119] Using mem_required_per_node from the hooks. Tested on Snellius and it works properly. --- eessi/testsuite/tests/apps/espresso/espresso.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 9fbccf58..2fbb5ce3 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -83,8 +83,9 @@ def set_num_tasks_per_node(self): def set_mem(self): """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at least 1 GB per core of memory.""" - mem_required_per_node = str(self.num_tasks_per_node * 1) + 'GB' - self.extra_resources = {'memory': {'size': mem_required_per_node}} + mem_required_per_node = self.num_tasks_per_node * 0.9 + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + @deferrable def assert_completion(self): From 140b9e424e81694f22207303eda7e54467d932b3 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Fri, 7 Jun 2024 00:03:31 +0200 Subject: [PATCH 074/119] Making the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 2fbb5ce3..7db09ff9 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -86,7 +86,6 @@ def set_mem(self): mem_required_per_node = self.num_tasks_per_node * 0.9 hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) - @deferrable def assert_completion(self): '''Check completion''' From dcd268810440fc458cab43f538faf8ecd473cea4 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Sat, 8 Jun 2024 11:47:24 +0200 Subject: [PATCH 075/119] fix memory per node --- config/vsc_hortense.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index f349bf60..1615330b 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -59,7 +59,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) + 'mem_per_node': 234 # in GiB }, }, { @@ -91,7 +91,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 511.983 # in GiB + 'mem_per_node': 473 # in GiB }, }, { @@ -123,7 +123,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 256.000 # in GiB (should be checked, its unclear from slurm.conf) + 'mem_per_node': 234 # in GiB }, }, { @@ -150,7 +150,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 256.000 # in GiB + 'mem_per_node': 236 # in GiB }, 'resources': [ { @@ -194,7 +194,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 511.983 # in GiB + 'mem_per_node': 475 # in GiB }, 'resources': [ { From 889e518deacb39efe8b0f94007c2477c14a6d0d6 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 9 Jun 2024 10:23:00 +0200 Subject: [PATCH 076/119] use MiB units for slurm --- eessi/testsuite/hooks.py | 43 +++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 5dd98a7f..c40613e4 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -383,7 +383,7 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic log(f'valid_systems set to {test.valid_systems}') -def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): +def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: int): """ This hook will request a specific amount of memory per node to the batch scheduler. First, it computes which fraction of CPUs is requested from a node, and how much the corresponding (proportional) @@ -396,59 +396,56 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req): Arguments: - test: the ReFrame test to which this hook should apply - - app_mem_req: the amount of memory this application needs (per node) in GiB + - app_mem_req: the amount of memory this application needs (per node) in MiB Example 1: - A system with 128 cores and 64 GiB per node. - The test is launched on 64 cores - - The app_mem_req is 40 (GiB) - In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32 GiB. - The app_mem_req is higher. Thus, 40GiB (per node) is requested from the batch scheduler. + - The app_mem_req is 40,000 (MiB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32,000 MiB. + The app_mem_req is higher. Thus, 40,000 MiB (per node) is requested from the batch scheduler. Example 2: - - A system with 128 cores per node, 128 GiB mem per node is used. + - A system with 128 cores per node, 128,000 MiB mem per node is used. - The test is launched on 64 cores - - the app_mem_req is 40 (GiB) - In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64 GiB. - This is higher than the app_mem_req. Thus, 64 GiB (per node) is requested from the batch scheduler. + - the app_mem_req is 40,000 (MiB) + In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64,000 MiB. + This is higher than the app_mem_req. Thus, 64,000 MiB (per node) is requested from the batch scheduler. """ # Check that the systems.partitions.extra dict in the ReFrame config contains mem_per_node check_extras_key_defined(test, 'mem_per_node') # Skip if the current partition doesn't have sufficient memory to run the application - msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} GiB" + msg = f"Skipping test: nodes in this partition only have {test.current_partition.extras['mem_per_node']} MiB" msg += " memory available (per node) accodring to the current ReFrame configuration," - msg += f" but {app_mem_req} GiB is needed" + msg += f" but {app_mem_req} MiB is needed" test.skip_if(test.current_partition.extras['mem_per_node'] < app_mem_req, msg) # Compute what is higher: the requested memory, or the memory available proportional to requested CPUs # Fraction of CPU cores requested check_proc_attribute_defined(test, 'num_cpus') cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus - proportional_mem = cpu_fraction * test.current_partition.extras['mem_per_node'] + proportional_mem = math.floor(cpu_fraction * test.current_partition.extras['mem_per_node']) scheduler_name = test.current_partition.scheduler.registered_name if scheduler_name == 'slurm' or scheduler_name == 'squeue': - # SLURMs --mem defines memory per node, see https://slurm.schedmd.com/sbatch.html - # SLURM uses megabytes and gigabytes, i.e. base-10, so conversion is 1000, not 1024 - # Thus, we convert from GiB (gibibytes) to MB (megabytes) (1024 * 1024 * 1024 / (1000 * 1000) = 1073.741824) - app_mem_req = math.ceil(1073.741824 * app_mem_req) - log(f"Memory requested by application: {app_mem_req} MB") - proportional_mem = math.floor(1073.741824 * proportional_mem) - log(f"Memory proportional to the core count: {proportional_mem} MB") + # SLURM defines --mem as memory per node, see https://slurm.schedmd.com/sbatch.html + # SLURM uses MiB units by default + log(f"Memory requested by application: {app_mem_req} MiB") + log(f"Memory proportional to the core count: {proportional_mem} MiB") # Request the maximum of the proportional_mem, and app_mem_req to the scheduler req_mem_per_node = max(proportional_mem, app_mem_req) test.extra_resources = {'memory': {'size': f'{req_mem_per_node}M'}} - log(f"Requested {req_mem_per_node} MB per node from the SLURM batch scheduler") + log(f"Requested {req_mem_per_node} MiB per node from the SLURM batch scheduler") elif scheduler_name == 'torque': # Torque/moab requires asking for --pmem (--mem only works single node and thus doesnt generalize) # See https://docs.adaptivecomputing.com/10-0-1/Torque/torque.htm#topics/torque/3-jobs/3.1.3-requestingRes.htm - # Units are MiB according to the documentation, thus, we simply multiply with 1024 + # Units are MiB according to the documentation # We immediately divide by num_tasks_per_node (before rounding), since -pmem specifies memroy _per process_ - app_mem_req_task = math.ceil(1024 * app_mem_req / test.num_tasks_per_node) - proportional_mem_task = math.floor(1024 * proportional_mem / test.num_tasks_per_node) + app_mem_req_task = math.ceil(app_mem_req / test.num_tasks_per_node) + proportional_mem_task = math.floor(proportional_mem / test.num_tasks_per_node) # Request the maximum of the proportional_mem, and app_mem_req to the scheduler req_mem_per_task = max(proportional_mem_task, app_mem_req_task) From 0bacba813ee58330597e7ff871c02c17d49ca190 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 9 Jun 2024 12:12:57 +0200 Subject: [PATCH 077/119] use MiB units for memory per node --- eessi/testsuite/hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index c40613e4..d6829d08 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -383,7 +383,7 @@ def filter_valid_systems_by_device_type(test: rfm.RegressionTest, required_devic log(f'valid_systems set to {test.valid_systems}') -def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: int): +def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: float): """ This hook will request a specific amount of memory per node to the batch scheduler. First, it computes which fraction of CPUs is requested from a node, and how much the corresponding (proportional) @@ -425,6 +425,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: int): check_proc_attribute_defined(test, 'num_cpus') cpu_fraction = test.num_tasks_per_node * test.num_cpus_per_task / test.current_partition.processor.num_cpus proportional_mem = math.floor(cpu_fraction * test.current_partition.extras['mem_per_node']) + app_mem_req = math.ceil(app_mem_req) scheduler_name = test.current_partition.scheduler.registered_name if scheduler_name == 'slurm' or scheduler_name == 'squeue': From e5234583ec01cda67927471554753a9b5043f20d Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Mon, 10 Jun 2024 16:25:55 +0200 Subject: [PATCH 078/119] Removing 16 node test case for now since it takes way too long and have dialing down the scales within the CI tests since they should not take too much time. --- eessi/testsuite/tests/apps/espresso/espresso.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 7db09ff9..7213ee6c 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -17,11 +17,23 @@ from eessi.testsuite.constants import * from eessi.testsuite.utils import find_modules, log +def filter_scales_P3M(): + """ + Filtering function for filtering scales for P3M test. + This is currently required because the 16 node test takes way too long and always fails due to time limit. + Once a solution to mesh tuning algorithm is found, where we can specify the mesh sizes for a particular scale, + this function can be removed. + """ + return [ + k for (k, v) in SCALES.items() + if v['num_nodes'] != 16 + ] + @rfm.simple_test class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): - scale = parameter(SCALES.keys()) + scale = parameter(filter_scales_P3M()) valid_prog_environs = ['default'] valid_systems = ['*'] time_limit = '300m' @@ -55,7 +67,8 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] + and SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From c5e02458a45e39047dd5a81c7ad7b1a20304a139 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Mon, 10 Jun 2024 16:32:29 +0200 Subject: [PATCH 079/119] Trying to make the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 7213ee6c..5366fe5b 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -17,6 +17,7 @@ from eessi.testsuite.constants import * from eessi.testsuite.utils import find_modules, log + def filter_scales_P3M(): """ Filtering function for filtering scales for P3M test. @@ -67,8 +68,8 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] - and SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] and + SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From df8873c69fcc4f7ff47b486b8fd8ebfc2fa9e9f0 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Mon, 10 Jun 2024 16:36:04 +0200 Subject: [PATCH 080/119] Making the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 5366fe5b..a1675afd 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -68,8 +68,7 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] and - SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] and SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From aebfdc189c5fa6104bc6a0e588d2914a32b9eab0 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 11 Jun 2024 16:55:28 +0200 Subject: [PATCH 081/119] Removing files that are not relevant: job.sh and benchmark.csv and removing the statement from madelung that puts benchmark.csv as path within the output parameter. --- .../tests/apps/espresso/benchmarks.csv | 34 ------------------- .../testsuite/tests/apps/espresso/src/job.sh | 10 ------ .../tests/apps/espresso/src/madelung.py | 3 -- 3 files changed, 47 deletions(-) delete mode 100644 eessi/testsuite/tests/apps/espresso/benchmarks.csv delete mode 100644 eessi/testsuite/tests/apps/espresso/src/job.sh diff --git a/eessi/testsuite/tests/apps/espresso/benchmarks.csv b/eessi/testsuite/tests/apps/espresso/benchmarks.csv deleted file mode 100644 index 9091534b..00000000 --- a/eessi/testsuite/tests/apps/espresso/benchmarks.csv +++ /dev/null @@ -1,34 +0,0 @@ -"mode","cores","mpi.x","mpi.y","mpi.z","particles","mean","std" -"weak scaling",4,2,2,1,6912,2.341e-01,8.081e-03 -"strong scaling",4,2,2,1,5832,2.496e-01,9.019e-03 -"weak scaling",16,4,2,2,27648,2.417e+00,9.576e-02 -"strong scaling",16,4,2,2,5832,3.853e-02,1.991e-03 -"weak scaling",32,4,4,2,55296,4.263e+00,1.161e+00 -"strong scaling",32,4,4,2,5832,2.194e-02,7.303e-04 -"weak scaling",1,1,1,1,1728,7.655e-02,3.434e-03 -"weak scaling",2,2,1,1,3456,1.456e-01,4.679e-03 -"strong scaling",2,2,1,1,5832,3.936e-01,1.098e-02 -"strong scaling",1,1,1,1,5832,6.333e-01,1.194e-01 -"strong scaling",64,4,4,4,5832,1.910e-02,6.132e-04 -"weak scaling",1,1,1,1,1728,9.482e-02,2.956e-03 -"weak scaling",2,2,1,1,3456,2.111e-01,6.614e-03 -"strong scaling",1,1,1,1,5832,9.133e-01,2.868e-02 -"strong scaling",16,4,2,2,5832,4.285e-02,1.327e-03 -"strong scaling",64,4,4,4,5832,1.715e-02,5.776e-04 -"strong scaling",128,8,4,4,5832,1.980e-02,7.013e-04 -"weak scaling",64,4,4,4,110592,4.375e-01,1.414e-02 -"weak scaling",100,5,5,4,172800,4.450e-01,1.437e-02 -"weak scaling",128,8,4,4,221184,8.720e+00,2.753e-01 -"weak scaling",128,8,4,4,221184,8.760e+00,3.110e-01 -"weak scaling",4,2,2,1,6912,2.626e-01,8.142e-03 -"weak scaling",4,2,2,1,6912,2.780e-01,8.683e-03 -"weak scaling",4,2,2,1,6912,2.627e-01,8.391e-03 -"weak scaling",4,2,2,1,6912,2.617e-01,8.155e-03 -"weak scaling",2,2,1,1,3456,2.028e-01,6.255e-03 -"weak scaling",2,2,1,1,3456,3.247e-01,1.026e-02 -"weak scaling",2,2,1,1,3456,3.249e-01,1.029e-02 -"weak scaling",2,2,1,1,3456,3.257e-01,1.028e-02 -"weak scaling",2,2,1,1,3456,3.375e-01,1.095e-02 -"weak scaling",2,2,1,1,3456,3.367e-01,1.086e-02 -"weak scaling",2,2,1,1,3456,3.241e-01,1.048e-02 -"weak scaling",2,2,1,1,3456,3.243e-01,1.038e-02 diff --git a/eessi/testsuite/tests/apps/espresso/src/job.sh b/eessi/testsuite/tests/apps/espresso/src/job.sh deleted file mode 100644 index 17399c52..00000000 --- a/eessi/testsuite/tests/apps/espresso/src/job.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -#SBATCH --time=00:40:00 -#SBATCH --output %j.stdout -#SBATCH --error %j.stderr -module load spack/default gcc/12.3.0 cuda/12.3.0 openmpi/4.1.6 \ - fftw/3.3.10 boost/1.83.0 python/3.12.1 -source ../espresso-4.3/venv/bin/activate -srun --cpu-bind=cores python3 madelung.py --size 6 --weak-scaling -srun --cpu-bind=cores python3 madelung.py --size 9 --strong-scaling -deactivate diff --git a/eessi/testsuite/tests/apps/espresso/src/madelung.py b/eessi/testsuite/tests/apps/espresso/src/madelung.py index 37d0b44a..3f73b5d5 100644 --- a/eessi/testsuite/tests/apps/espresso/src/madelung.py +++ b/eessi/testsuite/tests/apps/espresso/src/madelung.py @@ -34,9 +34,6 @@ default=False, required=False, help="Use GPU implementation") parser.add_argument("--topology", metavar=("X", "Y", "Z"), nargs=3, action="store", default=None, required=False, type=int, help="Cartesian topology") -parser.add_argument("--output", metavar="FILEPATH", action="store", - type=str, required=False, default="benchmarks.csv", - help="Output file (default: benchmarks.csv)") group = parser.add_mutually_exclusive_group() group.add_argument("--weak-scaling", action="store_true", help="Weak scaling benchmark (Gustafson's law: constant work per core)") From 8ccec25cb86379014f033dc7f62b38b72dcd599f Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 11 Jun 2024 20:57:13 +0200 Subject: [PATCH 082/119] also fix config files and QE hook --- config/github_actions.py | 2 +- config/it4i_karolina.py | 2 +- config/izum_vega.py | 4 ++-- config/surf_snellius.py | 6 +++--- config/vsc_hortense.py | 10 +++++----- eessi/testsuite/tests/apps/QuantumESPRESSO.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/config/github_actions.py b/config/github_actions.py index b2196b6b..7ed97422 100644 --- a/config/github_actions.py +++ b/config/github_actions.py @@ -33,7 +33,7 @@ # Make sure to round down, otherwise a job might ask for more mem than is available # per node # This is a fictional amount, GH actions probably has less, but only does --dry-run - 'mem_per_node': 30 # in GiB + 'mem_per_node': 30 * 1024 # in MiB }, } ] diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py index 2bdfa035..4904bf1d 100644 --- a/config/it4i_karolina.py +++ b/config/it4i_karolina.py @@ -62,7 +62,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 219.345 # in GiB + 'mem_per_node': 219.345 * 1024 # in MiB }, 'descr': 'CPU Universal Compute Nodes, see https://docs.it4i.cz/karolina/hardware-overview/' }, diff --git a/config/izum_vega.py b/config/izum_vega.py index f7193aed..e3b53752 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -62,7 +62,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 238.418 # in GiB + 'mem_per_node': 238.418 * 1024 # in MiB }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, @@ -106,7 +106,7 @@ # 'extras': { # # Make sure to round down, otherwise a job might ask for more mem than is available # # per node - # 'mem_per_node': 476.837 # in GiB (should be checked, its unclear from slurm.conf) + # 'mem_per_node': 476.837 * 1024 # in MiB (should be checked, its unclear from slurm.conf) # }, # 'descr': 'GPU partition, see https://en-doc.vega.izum.si/architecture/' # }, diff --git a/config/surf_snellius.py b/config/surf_snellius.py index d8bcc36c..c4c0623a 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -56,7 +56,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 213.623 # in GiB + 'mem_per_node': 213.623 * 1024 # in MiB }, 'descr': 'AMD Rome CPU partition with native EESSI stack' }, @@ -80,7 +80,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 320.434 # in GiB + 'mem_per_node': 320.434 * 1024 # in MiB }, 'descr': 'AMD Genoa CPU partition with native EESSI stack' }, @@ -117,7 +117,7 @@ GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 457.763 # in GiB + 'mem_per_node': 457.763 * 1024 # in MiB }, 'descr': 'Nvidia A100 GPU partition with native EESSI stack' }, diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index 1615330b..9d52d1c9 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -59,7 +59,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 234 # in GiB + 'mem_per_node': 234 * 1024 # in MiB }, }, { @@ -91,7 +91,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 473 # in GiB + 'mem_per_node': 473 * 1024 # in MiB }, }, { @@ -123,7 +123,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 234 # in GiB + 'mem_per_node': 234 * 1024 # in MiB }, }, { @@ -150,7 +150,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 236 # in GiB + 'mem_per_node': 236 * 1024 # in MiB }, 'resources': [ { @@ -194,7 +194,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 475 # in GiB + 'mem_per_node': 475 * 1024 # in MiB }, 'resources': [ { diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index 050e43d3..288354b2 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -100,7 +100,7 @@ def run_after_setup(self): @run_after('setup') def request_mem(self): memory_required = self.num_tasks_per_node * 0.9 + 4 - hooks.req_memory_per_node(test=self, app_mem_req=memory_required) + hooks.req_memory_per_node(test=self, app_mem_req=memory_required * 1024) @run_after('setup') def set_omp_num_threads(self): From 13a6312cf5250b9b7d3e6ada7c1310af39002bce Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 11 Jun 2024 21:07:18 +0200 Subject: [PATCH 083/119] update hortense config according to slurm.conf --- config/vsc_hortense.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/vsc_hortense.py b/config/vsc_hortense.py index 9d52d1c9..312c72f4 100644 --- a/config/vsc_hortense.py +++ b/config/vsc_hortense.py @@ -59,7 +59,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 234 * 1024 # in MiB + 'mem_per_node': 252160, # in MiB }, }, { @@ -91,7 +91,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 473 * 1024 # in MiB + 'mem_per_node': 508160, # in MiB }, }, { @@ -123,7 +123,7 @@ def command(self, job): 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 234 * 1024 # in MiB + 'mem_per_node': 252160, # in MiB }, }, { @@ -150,7 +150,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 236 * 1024 # in MiB + 'mem_per_node': 254400, # in MiB }, 'resources': [ { @@ -194,7 +194,7 @@ def command(self, job): GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 475 * 1024 # in MiB + 'mem_per_node': 510720, # in MiB }, 'resources': [ { From 79d604b563e60bfb16b7c25b7008f91532d8a7bb Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 11 Jun 2024 22:57:23 +0200 Subject: [PATCH 084/119] also update espresso test --- eessi/testsuite/tests/apps/espresso/espresso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index a1675afd..20ea5e7e 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -97,7 +97,7 @@ def set_mem(self): """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at least 1 GB per core of memory.""" mem_required_per_node = self.num_tasks_per_node * 0.9 - hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node * 1024) @deferrable def assert_completion(self): From 8075ae9b2f8159f079457ad9a50e85108cf8b60b Mon Sep 17 00:00:00 2001 From: Sam Moors Date: Thu, 13 Jun 2024 13:58:43 +0200 Subject: [PATCH 085/119] fix comment Co-authored-by: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index d6829d08..c4d658df 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -399,7 +399,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: float): - app_mem_req: the amount of memory this application needs (per node) in MiB Example 1: - - A system with 128 cores and 64 GiB per node. + - A system with 128 cores and 64,000 MiB per node. - The test is launched on 64 cores - The app_mem_req is 40,000 (MiB) In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 32,000 MiB. From 96f6b9d136b5298ca2c1dd1b5846b2935581d467 Mon Sep 17 00:00:00 2001 From: Sam Moors Date: Thu, 13 Jun 2024 14:00:05 +0200 Subject: [PATCH 086/119] fix comment Co-authored-by: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index c4d658df..ab711955 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -406,7 +406,7 @@ def req_memory_per_node(test: rfm.RegressionTest, app_mem_req: float): The app_mem_req is higher. Thus, 40,000 MiB (per node) is requested from the batch scheduler. Example 2: - - A system with 128 cores per node, 128,000 MiB mem per node is used. + - A system with 128 cores per node, 128,000 MiB mem per node. - The test is launched on 64 cores - the app_mem_req is 40,000 (MiB) In this case, the test requests 50% of the CPUs. Thus, the proportional amount of memory is 64,000 MiB. From bdd813c7dab1fe629a28915d9eb70fb0ea32f382 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:36:14 +0200 Subject: [PATCH 087/119] Added / updated memory for various systems in MiB units --- config/aws_mc.py | 5 +++++ config/it4i_karolina.py | 2 +- config/izum_vega.py | 4 +++- config/surf_snellius.py | 6 +++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/aws_mc.py b/config/aws_mc.py index 4419ea27..2cf5e3f4 100644 --- a/config/aws_mc.py +++ b/config/aws_mc.py @@ -105,6 +105,11 @@ # steps inherit environment. It doesn't hurt to define this even if srun is not used 'export SLURM_EXPORT_ENV=ALL' ], + 'extras': { + # Node types have somewhat varying amounts of memory, but we'll make it easy on ourselves + # All should _at least_ have this amount (30GB * 1E9 / (1024*1024) = 28610 MiB) + 'mem_per_node': 28610 + }, } for system in site_configuration['systems']: for partition in system['partitions']: diff --git a/config/it4i_karolina.py b/config/it4i_karolina.py index 4904bf1d..b9dae87c 100644 --- a/config/it4i_karolina.py +++ b/config/it4i_karolina.py @@ -62,7 +62,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 219.345 * 1024 # in MiB + 'mem_per_node': 235520 # in MiB }, 'descr': 'CPU Universal Compute Nodes, see https://docs.it4i.cz/karolina/hardware-overview/' }, diff --git a/config/izum_vega.py b/config/izum_vega.py index e3b53752..4c95c2d8 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -62,7 +62,9 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 238.418 * 1024 # in MiB + # NB: Vega's MaxMemPerNode is set to 256000, but this MUST be a MB/MiB units mistake + # Most likely, it is 256 GB, so 256*1E9/(1024*1024) MiB + 'mem_per_node': 244140 # in MiB }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, diff --git a/config/surf_snellius.py b/config/surf_snellius.py index c4c0623a..827ca32c 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -56,7 +56,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 213.623 * 1024 # in MiB + 'mem_per_node': 229376 # in MiB }, 'descr': 'AMD Rome CPU partition with native EESSI stack' }, @@ -80,7 +80,7 @@ 'extras': { # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 320.434 * 1024 # in MiB + 'mem_per_node': 344064 # in MiB }, 'descr': 'AMD Genoa CPU partition with native EESSI stack' }, @@ -117,7 +117,7 @@ GPU_VENDOR: GPU_VENDORS[NVIDIA], # Make sure to round down, otherwise a job might ask for more mem than is available # per node - 'mem_per_node': 457.763 * 1024 # in MiB + 'mem_per_node': 491520 # in MiB }, 'descr': 'Nvidia A100 GPU partition with native EESSI stack' }, From 0a886c106cd54e1db573bcf745fa39543fd4c8c2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 13 Jun 2024 14:42:14 +0200 Subject: [PATCH 088/119] Make linter happy --- config/izum_vega.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/izum_vega.py b/config/izum_vega.py index 4c95c2d8..939c089c 100644 --- a/config/izum_vega.py +++ b/config/izum_vega.py @@ -64,7 +64,7 @@ # per node # NB: Vega's MaxMemPerNode is set to 256000, but this MUST be a MB/MiB units mistake # Most likely, it is 256 GB, so 256*1E9/(1024*1024) MiB - 'mem_per_node': 244140 # in MiB + 'mem_per_node': 244140 # in MiB }, 'descr': 'CPU partition Standard, see https://en-doc.vega.izum.si/architecture/' }, From 7c6f347ed9e57b002517518b3e9a45b76829831c Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:03:29 +0200 Subject: [PATCH 089/119] Adding lj test. This scales slightly better and takes less time compared to the P3M test. --- .../testsuite/tests/apps/espresso/espresso.py | 31 +++- eessi/testsuite/tests/apps/espresso/src/lj.py | 162 ++++++++++++++++++ 2 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 eessi/testsuite/tests/apps/espresso/src/lj.py diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index a1675afd..89168334 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -32,7 +32,7 @@ def filter_scales_P3M(): @rfm.simple_test -class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): +class EESSI_ESPRESSO(rfm.RunOnlyRegressionTest): scale = parameter(filter_scales_P3M()) valid_prog_environs = ['default'] @@ -50,6 +50,7 @@ class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(rfm.RunOnlyRegressionTest): benchmark_info = parameter([ ('mpi.ionic_crystals.p3m', 'p3m'), + ('mpi.particles.lj', 'lj'), ], fmt=lambda x: x[0], loggable=True) @run_after('init') @@ -75,16 +76,25 @@ def set_tag_ci(self): if (self.benchmark_info[0] == 'mpi.ionic_crystals.p3m'): self.tags.add('ionic_crystals_p3m') + if (self.benchmark_info[0] == 'mpi.particles.lj'): + self.tags.add('particles_lj') + @run_after('init') def set_executable_opts(self): """Set executable opts based on device_type parameter""" num_default = 0 # If this test already has executable opts, they must have come from the command line hooks.check_custom_executable_opts(self, num_default=num_default) - if not self.has_custom_executable_opts: + if (not self.has_custom_executable_opts and self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']): # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a # corresponding min node size has to be chozen. self.executable_opts += ['--size', str(self.default_weak_scaling_system_size), '--weak-scaling'] utils.log(f'executable_opts set to {self.executable_opts}') + elif (not self.has_custom_executable_opts and self.benchmark_info[0] in ['mpi.particles.lj']): + # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a + # corresponding min node size has to be chozen. For this test the default values embedded in the lj.py are + # good enough. Otherwise custom executable options can be passed anyways. + self.executable = 'python3 lj.py' # Updating the executable. + @run_after('setup') def set_num_tasks_per_node(self): @@ -102,14 +112,23 @@ def set_mem(self): @deferrable def assert_completion(self): '''Check completion''' - cao = sn.extractsingle(r'^resulting parameters:.*cao: (?P\S+),', self.stdout, 'cao', int) - return (sn.assert_found(r'^Algorithm executed.', self.stdout) and cao) + if self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']: + cao = sn.extractsingle(r'^resulting parameters:.*cao: (?P\S+),', self.stdout, 'cao', int) + return (sn.assert_found(r'^Algorithm executed.', self.stdout) and cao) + elif self.benchmark_info[0] in ['mpi.particles.lj']: + return (sn.assert_found(r'^Algorithm executed.', self.stdout)) @deferrable def assert_convergence(self): '''Check convergence''' - check_string = sn.assert_found(r'Final convergence met with tolerances:', self.stdout) - energy = sn.extractsingle(r'^\s+energy:\s+(?P\S+)', self.stdout, 'energy', float) + check_string = False + energy = 0.0 + if self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']: + check_string = sn.assert_found(r'Final convergence met with tolerances:', self.stdout) + energy = sn.extractsingle(r'^\s+energy:\s+(?P\S+)', self.stdout, 'energy', float) + elif self.benchmark_info[0] in ['mpi.particles.lj']: + check_string = sn.assert_found(r'Final convergence met with relative tolerances:', self.stdout) + energy = sn.extractsingle(r'^\s+sim_energy:\s+(?P\S+)', self.stdout, 'energy', float) return (check_string and (energy != 0.0)) @sanity_function diff --git a/eessi/testsuite/tests/apps/espresso/src/lj.py b/eessi/testsuite/tests/apps/espresso/src/lj.py new file mode 100644 index 00000000..366b71aa --- /dev/null +++ b/eessi/testsuite/tests/apps/espresso/src/lj.py @@ -0,0 +1,162 @@ +# +# Copyright (C) 2018-2024 The ESPResSo project +# +# This file is part of ESPResSo. +# +# ESPResSo is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ESPResSo is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +import argparse +import time +import espressomd +import numpy as np + +required_features = ["LENNARD_JONES"] +espressomd.assert_features(required_features) + +parser = argparse.ArgumentParser(description="Benchmark LJ simulations.") +parser.add_argument("--particles-per-core", metavar="N", action="store", + type=int, default=2000, required=False, + help="Number of particles in the simulation box") +parser.add_argument("--sample-size", metavar="S", action="store", + type=int, default=30, required=False, + help="Sample size") +parser.add_argument("--volume-fraction", metavar="FRAC", action="store", + type=float, default=0.50, required=False, + help="Fraction of the simulation box volume occupied by " + "particles (range: [0.01-0.74], default: 0.50)") +args = parser.parse_args() + +# process and check arguments +measurement_steps = 100 +if args.particles_per_core < 16000: + measurement_steps = 200 +if args.particles_per_core < 10000: + measurement_steps = 500 +if args.particles_per_core < 5000: + measurement_steps = 1000 +if args.particles_per_core < 1000: + measurement_steps = 2000 +if args.particles_per_core < 600: + measurement_steps = 4000 +if args.particles_per_core < 260: + measurement_steps = 6000 +assert args.volume_fraction > 0., "volume_fraction must be a positive number" +assert args.volume_fraction < np.pi / (3. * np.sqrt(2.)), \ + "volume_fraction exceeds the physical limit of sphere packing (~0.74)" + +# make simulation deterministic +np.random.seed(42) + + +def get_reference_values_per_atom(x): + # result of a polynomial fit in the range from 0.01 to 0.55 + energy = 54.2 * x**3 - 23.8 * x**2 + 4.6 * x - 0.09 + pressure = 377. * x**3 - 149. * x**2 + 32.2 * x - 0.58 + return energy, pressure + + +def get_normalized_values_per_atom(system): + energy = system.analysis.energy()["non_bonded"] + pressure = system.analysis.pressure()["non_bonded"] + N = len(system.part) + V = system.volume() + return 2. * energy / N, 2. * pressure * V / N + + +system = espressomd.System(box_l=[10., 10., 10.]) +system.time_step = 0.01 +system.cell_system.skin = 0.5 + +lj_eps = 1.0 # LJ epsilon +lj_sig = 1.0 # particle diameter +lj_cut = lj_sig * 2**(1. / 6.) # cutoff distance + +n_proc = system.cell_system.get_state()["n_nodes"] +n_part = n_proc * args.particles_per_core +node_grid = np.array(system.cell_system.node_grid) +# volume of N spheres with radius r: N * (4/3*pi*r^3) +box_v = args.particles_per_core * 4. / 3. * \ + np.pi * (lj_sig / 2.)**3 / args.volume_fraction +# box_v = (x * n) * x * x for a column +system.box_l = float((box_v)**(1. / 3.)) * node_grid +assert np.abs(n_part * 4. / 3. * np.pi * (lj_sig / 2.)**3 / + np.prod(system.box_l) - args.volume_fraction) < 0.1 + +system.non_bonded_inter[0, 0].lennard_jones.set_params( + epsilon=lj_eps, sigma=lj_sig, cutoff=lj_cut, shift="auto") + +system.part.add(pos=np.random.random((n_part, 3)) * system.box_l) + +# energy minimization +max_steps = 1000 +# particle forces for volume fractions between 0.1 and 0.5 follow a polynomial +target_f_max = 20. * args.volume_fraction**2 +system.integrator.set_steepest_descent( + f_max=target_f_max, gamma=0.001, max_displacement=0.01 * lj_sig) +n_steps = system.integrator.run(max_steps) +assert n_steps < max_steps, f'''energy minimization failed: \ +E = {system.analysis.energy()["total"] / len(system.part):.3g} per particle, \ +f_max = {np.max(np.linalg.norm(system.part.all().f, axis=1)):.2g}, \ +target f_max = {target_f_max:.2g}''' + +# warmup +system.integrator.set_vv() +system.thermostat.set_langevin(kT=1.0, gamma=1.0, seed=42) + +# tuning and equilibration +min_skin = 0.2 +max_skin = 1.0 +print("Tune skin: {:.3f}".format(system.cell_system.tune_skin( + min_skin=min_skin, max_skin=max_skin, tol=0.05, int_steps=100))) +print("Equilibration") +system.integrator.run(min(5 * measurement_steps, 60000)) +print("Tune skin: {:.3f}".format(system.cell_system.tune_skin( + min_skin=min_skin, max_skin=max_skin, tol=0.05, int_steps=100))) +print("Equilibration") +system.integrator.run(min(10 * measurement_steps, 60000)) + +print("Sampling runtime...") +timings = [] +energies = [] +pressures = [] +for i in range(args.sample_size): + tick = time.time() + system.integrator.run(measurement_steps) + tock = time.time() + t = (tock - tick) / measurement_steps + timings.append(t) + energy, pressure = get_normalized_values_per_atom(system) + energies.append(energy) + pressures.append(pressure) + +sim_energy = np.mean(energies) +sim_pressure = np.mean(pressures) +ref_energy, ref_pressure = get_reference_values_per_atom(args.volume_fraction) + +print("Algorithm executed. \n") +np.testing.assert_allclose(sim_energy, ref_energy, atol=0., rtol=0.1) +np.testing.assert_allclose(sim_pressure, ref_pressure, atol=0., rtol=0.1) + +print("Final convergence met with relative tolerances: \n\ + sim_energy: ", 0.1, "\n\ + sim_pressure: ", 0.1, "\n") + +header = '"mode","cores","mpi.x","mpi.y","mpi.z","particles","volume_fraction","mean","std"' +report = f'''"weak scaling",{n_proc},{node_grid[0]},{node_grid[1]},\ +{node_grid[2]},{len(system.part)},{args.volume_fraction:.4f},\ +{np.mean(timings):.3e},{np.std(timings,ddof=1):.3e}''' +print(header) +print(report) +print(f"Performance: {np.mean(timings):.3e}") From e88ff15bc368fd9b74f23f6d28bb3c3b8ec1c700 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:07:31 +0200 Subject: [PATCH 090/119] Making the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 3 +-- eessi/testsuite/tests/apps/espresso/src/lj.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 89168334..3729aa97 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -93,8 +93,7 @@ def set_executable_opts(self): # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a # corresponding min node size has to be chozen. For this test the default values embedded in the lj.py are # good enough. Otherwise custom executable options can be passed anyways. - self.executable = 'python3 lj.py' # Updating the executable. - + self.executable = 'python3 lj.py' # Updating the executable. @run_after('setup') def set_num_tasks_per_node(self): diff --git a/eessi/testsuite/tests/apps/espresso/src/lj.py b/eessi/testsuite/tests/apps/espresso/src/lj.py index 366b71aa..ad924b46 100644 --- a/eessi/testsuite/tests/apps/espresso/src/lj.py +++ b/eessi/testsuite/tests/apps/espresso/src/lj.py @@ -91,8 +91,7 @@ def get_normalized_values_per_atom(system): np.pi * (lj_sig / 2.)**3 / args.volume_fraction # box_v = (x * n) * x * x for a column system.box_l = float((box_v)**(1. / 3.)) * node_grid -assert np.abs(n_part * 4. / 3. * np.pi * (lj_sig / 2.)**3 / - np.prod(system.box_l) - args.volume_fraction) < 0.1 +assert np.abs(n_part * 4. / 3. * np.pi * (lj_sig / 2.)**3 / np.prod(system.box_l) - args.volume_fraction) < 0.1 system.non_bonded_inter[0, 0].lennard_jones.set_params( epsilon=lj_eps, sigma=lj_sig, cutoff=lj_cut, shift="auto") From 15f044fd360cb1445c6c6a77a50d5c4b0e3609aa Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:09:47 +0200 Subject: [PATCH 091/119] Also adding LJ to the CI tag. --- eessi/testsuite/tests/apps/espresso/espresso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 3729aa97..b93c2442 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -69,7 +69,7 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m'] and SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From a481a123da4f74d2f17039f78d70bfdbc35d07e0 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:11:30 +0200 Subject: [PATCH 092/119] Making the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index b93c2442..092a4eb8 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -69,7 +69,8 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and + SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From 8593bf0dc972ab6f4f67c78655dce12cf1d8016a Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:13:21 +0200 Subject: [PATCH 093/119] Making the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 092a4eb8..7bf36d87 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -69,8 +69,8 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and - SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and \ + SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From e1126ebc3f6e74a8cb0241c1fd37bc7bbf7bdca8 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:14:44 +0200 Subject: [PATCH 094/119] Trying to make the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 7bf36d87..1b3529a1 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -69,7 +69,7 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and \ + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From 1e56b2c6a531a80019a254424ed745ef0b8b37cd Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Wed, 19 Jun 2024 21:16:45 +0200 Subject: [PATCH 095/119] Trying to make the linter happy. --- eessi/testsuite/tests/apps/espresso/espresso.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index 1b3529a1..f36a8148 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -69,8 +69,8 @@ def run_after_init(self): @run_after('init') def set_tag_ci(self): """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] and - SCALES[self.scale]['num_nodes'] < 2): + if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] + and SCALES[self.scale]['num_nodes'] < 2): self.tags.add('CI') log(f'tags set to {self.tags}') From f816f4ec208405df7b63a759563f76809bd62392 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Tue, 25 Jun 2024 15:28:17 +0200 Subject: [PATCH 096/119] Split the ESPResSo tests into 2 classes --- .../testsuite/tests/apps/espresso/espresso.py | 139 ++++++++++++------ 1 file changed, 95 insertions(+), 44 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index f36a8148..d80f6790 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -31,28 +31,14 @@ def filter_scales_P3M(): ] -@rfm.simple_test class EESSI_ESPRESSO(rfm.RunOnlyRegressionTest): - - scale = parameter(filter_scales_P3M()) valid_prog_environs = ['default'] valid_systems = ['*'] - time_limit = '300m' # Need to check if QuantumESPRESSO also gets listed. module_name = parameter(find_modules('ESPResSo')) # device type is parameterized for an impending CUDA ESPResSo module. device_type = parameter([DEVICE_TYPES[CPU]]) - executable = 'python3 madelung.py' - - default_strong_scaling_system_size = 9 - default_weak_scaling_system_size = 6 - - benchmark_info = parameter([ - ('mpi.ionic_crystals.p3m', 'p3m'), - ('mpi.particles.lj', 'lj'), - ], fmt=lambda x: x[0], loggable=True) - @run_after('init') def run_after_init(self): """hooks to run after init phase""" @@ -66,36 +52,6 @@ def run_after_init(self): # Set scales as tags hooks.set_tag_scale(self) - @run_after('init') - def set_tag_ci(self): - """ Setting tests under CI tag. """ - if (self.benchmark_info[0] in ['mpi.ionic_crystals.p3m', 'mpi.particles.lj'] - and SCALES[self.scale]['num_nodes'] < 2): - self.tags.add('CI') - log(f'tags set to {self.tags}') - - if (self.benchmark_info[0] == 'mpi.ionic_crystals.p3m'): - self.tags.add('ionic_crystals_p3m') - - if (self.benchmark_info[0] == 'mpi.particles.lj'): - self.tags.add('particles_lj') - - @run_after('init') - def set_executable_opts(self): - """Set executable opts based on device_type parameter""" - num_default = 0 # If this test already has executable opts, they must have come from the command line - hooks.check_custom_executable_opts(self, num_default=num_default) - if (not self.has_custom_executable_opts and self.benchmark_info[0] in ['mpi.ionic_crystals.p3m']): - # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a - # corresponding min node size has to be chozen. - self.executable_opts += ['--size', str(self.default_weak_scaling_system_size), '--weak-scaling'] - utils.log(f'executable_opts set to {self.executable_opts}') - elif (not self.has_custom_executable_opts and self.benchmark_info[0] in ['mpi.particles.lj']): - # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a - # corresponding min node size has to be chozen. For this test the default values embedded in the lj.py are - # good enough. Otherwise custom executable options can be passed anyways. - self.executable = 'python3 lj.py' # Updating the executable. - @run_after('setup') def set_num_tasks_per_node(self): """ Setting number of tasks per node and cpus per task in this function. This function sets num_cpus_per_task @@ -142,3 +98,98 @@ def assert_sanity(self): @performance_function('s/step') def perf(self): return sn.extractsingle(r'^Performance:\s+(?P\S+)', self.stdout, 'perf', float) + + +@rfm.simple_test +class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(EESSI_ESPRESSO): + scale = parameter(filter_scales_P3M()) + time_limit = '300m' + + executable = 'python3 madelung.py' + + default_weak_scaling_system_size = 6 + + @run_after('init') + def set_tag_ci(self): + """ Setting tests under CI tag. """ + if SCALES[self.scale]['num_nodes'] < 2: + self.tags.add('CI') + log(f'tags set to {self.tags}') + + self.tags.add('ionic_crystals_p3m') + + @run_after('init') + def set_executable_opts(self): + """Set executable opts based on device_type parameter""" + num_default = 0 # If this test already has executable opts, they must have come from the command line + hooks.check_custom_executable_opts(self, num_default=num_default) + # By default we run weak scaling since the strong scaling sizes need to change based on max node size and a + # corresponding min node size has to be chozen. + self.executable_opts += ['--size', str(self.default_weak_scaling_system_size), '--weak-scaling'] + utils.log(f'executable_opts set to {self.executable_opts}') + + @run_after('setup') + def set_mem(self): + """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at + least 1 GB per core of memory.""" + mem_required_per_node = self.num_tasks_per_node * 0.9 + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + + @deferrable + def assert_completion(self): + '''Check completion''' + cao = sn.extractsingle(r'^resulting parameters:.*cao: (?P\S+),', self.stdout, 'cao', int) + return (sn.assert_found(r'^Algorithm executed.', self.stdout) and cao) + + @deferrable + def assert_convergence(self): + '''Check convergence''' + check_string = False + energy = 0.0 + check_string = sn.assert_found(r'Final convergence met with tolerances:', self.stdout) + energy = sn.extractsingle(r'^\s+energy:\s+(?P\S+)', self.stdout, 'energy', float) + return (check_string and (energy != 0.0)) + + +@rfm.simple_test +class EESSI_ESPRESSO_LJ_PARTICLES(EESSI_ESPRESSO): + scale = parameter(SCALES.keys()) + time_limit = '300m' + + executable = 'python3 lj.py' + + @run_after('init') + def set_tag_ci(self): + """ Setting tests under CI tag. """ + if SCALES[self.scale]['num_nodes'] < 2: + self.tags.add('CI') + log(f'tags set to {self.tags}') + + self.tags.add('particles_lj') + + @run_after('init') + def set_executable_opts(self): + """Allow executable opts to be overwritten from command line""" + num_default = 0 # If this test already has executable opts, they must have come from the command line + hooks.check_custom_executable_opts(self, num_default=num_default) + + @run_after('setup') + def set_mem(self): + """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at + least 1 GB per core of memory.""" + mem_required_per_node = self.num_tasks_per_node * 0.9 # TODO: figure out if this is also ok for lb use case + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + + @deferrable + def assert_completion(self): + '''Check completion''' + return (sn.assert_found(r'^Algorithm executed.', self.stdout)) + + @deferrable + def assert_convergence(self): + '''Check convergence''' + check_string = False + energy = 0.0 + check_string = sn.assert_found(r'Final convergence met with relative tolerances:', self.stdout) + energy = sn.extractsingle(r'^\s+sim_energy:\s+(?P\S+)', self.stdout, 'energy', float) + return (check_string and (energy != 0.0)) From 4d8a8c11f608ca6c770314f92784647bc39ce730 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 27 Jun 2024 00:00:43 +0200 Subject: [PATCH 097/119] Setting lower memory requirements for LJ and applying scale filter to LJ * Setting the memory to 0.3 GiB per core. * The 16 node LJ test does not finish in 5 hours, thus applying the scale filter to it as well removing the 16_nodes scale for now. --- eessi/testsuite/tests/apps/espresso/espresso.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index d80f6790..bbc99ab1 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -18,9 +18,9 @@ from eessi.testsuite.utils import find_modules, log -def filter_scales_P3M(): +def filter_scales(): """ - Filtering function for filtering scales for P3M test. + Filtering function for filtering scales for P3M test and the LJ test. This is currently required because the 16 node test takes way too long and always fails due to time limit. Once a solution to mesh tuning algorithm is found, where we can specify the mesh sizes for a particular scale, this function can be removed. @@ -102,7 +102,7 @@ def perf(self): @rfm.simple_test class EESSI_ESPRESSO_P3M_IONIC_CRYSTALS(EESSI_ESPRESSO): - scale = parameter(filter_scales_P3M()) + scale = parameter(filter_scales()) time_limit = '300m' executable = 'python3 madelung.py' @@ -153,7 +153,7 @@ def assert_convergence(self): @rfm.simple_test class EESSI_ESPRESSO_LJ_PARTICLES(EESSI_ESPRESSO): - scale = parameter(SCALES.keys()) + scale = parameter(filter_scales()) time_limit = '300m' executable = 'python3 lj.py' @@ -176,8 +176,9 @@ def set_executable_opts(self): @run_after('setup') def set_mem(self): """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at - least 1 GB per core of memory.""" - mem_required_per_node = self.num_tasks_per_node * 0.9 # TODO: figure out if this is also ok for lb use case + least 1 GB per core of memory. LJ requires much lesser memory than P3M. 200 MB per core is as per measurement, + therefore 300 should be more than enough. """ + mem_required_per_node = self.num_tasks_per_node * 0.3 hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) @deferrable From 051d227309c5071222f2c9c25bd8efe07809417b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 27 Jun 2024 15:37:12 +0200 Subject: [PATCH 098/119] bump version to 0.3.0 + update release notes --- RELEASE_NOTES | 23 +++++++++++++++++++++++ pyproject.toml | 2 +- setup.cfg | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 79bd3dab..1b7a2088 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,29 @@ This file contains a description of the major changes to the EESSI test suite. For more detailed information, please see the git log. +v0.3.0 (27 June 2024) +--------------------- + +This is a minor release of the EESSI test-suite + +It includes: + +* Update config AWS MC cluster to use `software.eessi.io` (#126) +* Add test for QuantumESPRESSO (pw.x) (#128) +* Fix compact process binding for OpenMPI mpirun (#137) +* Use compact process binding for GROMACS (#139) +* Rename scale tags 1_cpn_2_nodes and 1_cpn_4_nodes (#140) +* Set SRUN_CPUS_PER_TASK for srun launcher (#141) +* Fix for "Failed to modify UD QP to INIT on mlx5_0" on Karolina CI runs (#142) +* Reduce the iteration count to make the OSU tests run faster, especially on slower interconnects (#143) +* Add test for ESPResSo (P3M) (#144) +* Use software.eessi.io repo in CI (#146) +* Add notes on release management to README (#148) +* Fix memory_per_node for Hortense (#151) +* Use MiB units for memory per node (#152) +* Added / updated memory for various systems in MiB units (#153) +* Add additional test for ESPRESSO (LJ) (#155) + v0.2.0 (7 March 2024) --------------------- diff --git a/pyproject.toml b/pyproject.toml index 2b3b607c..cfdefeeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "eessi-testsuite" -version = "0.2.0" +version = "0.3.0" description = "Test suite for the EESSI software stack" readme = "README.md" license = {file = "LICENSE"} diff --git a/setup.cfg b/setup.cfg index 49a7b178..28c7fc13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eessi-testsuite -version = 0.2.0 +version = 0.3.0 description = Test suite for the EESSI software stack long_description = file: README.md long_description_content_type = text/markdown From 1a4036b822765dbf519d1c1a47848a5e15f4ac1b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 27 Jun 2024 16:54:32 +0200 Subject: [PATCH 099/119] Add some extra instructions to bump the CI version before release --- CI/run_reframe.sh | 2 +- README.md | 1 + RELEASE_NOTES | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 2068f03f..57f209ca 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -48,7 +48,7 @@ if [ -z "${EESSI_TESTSUITE_URL}" ]; then EESSI_TESTSUITE_URL='https://github.com/EESSI/test-suite.git' fi if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then - EESSI_TESTSUITE_BRANCH='v0.2.0' + EESSI_TESTSUITE_BRANCH='v0.3.0' fi if [ -z "${EESSI_CVMFS_REPO}" ]; then export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io diff --git a/README.md b/README.md index 72878010..340087a4 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ are ok with that_ before doing so! When a release of the EESSI test suite is made, the following things must be taken care of: - Version bump: in both `pyproject.toml` and `setup.cfg`; +- Version bump the default `EESSI_TESTSUITE_BRANCH` in `CI/run_reframe.sh`; - Release notes: in `RELEASE_NOTES` + in GitHub release (cfr. https://github.com/EESSI/test-suite/releases/tag/v0.2.0); - Tag release on GitHub + publish release (incl. release notes); - Publishing release to PyPI: diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 1b7a2088..03b4c58d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -23,6 +23,7 @@ It includes: * Use MiB units for memory per node (#152) * Added / updated memory for various systems in MiB units (#153) * Add additional test for ESPRESSO (LJ) (#155) +* Bump default version used in CI (#157) v0.2.0 (7 March 2024) --------------------- From 77198cec325d0e0be2e010bc280a7ad85e54716d Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 27 Jun 2024 17:31:40 +0200 Subject: [PATCH 100/119] Correcting the memory to MiB --- eessi/testsuite/tests/apps/espresso/espresso.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index a87b22d1..e415201a 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -133,7 +133,7 @@ def set_mem(self): """ Setting an extra job option of memory. Here the assumption made is that HPC systems will contain at least 1 GB per core of memory.""" mem_required_per_node = self.num_tasks_per_node * 0.9 - hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node * 1024) @deferrable def assert_completion(self): @@ -179,7 +179,7 @@ def set_mem(self): least 1 GB per core of memory. LJ requires much lesser memory than P3M. 200 MB per core is as per measurement, therefore 300 should be more than enough. """ mem_required_per_node = self.num_tasks_per_node * 0.3 - hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node) + hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node * 1024) @deferrable def assert_completion(self): From 3cd399c565ec38bc679c8de4910788482ec7ae83 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 28 Jun 2024 12:09:30 +0200 Subject: [PATCH 101/119] bump version to 0.3.1 + update release notes --- RELEASE_NOTES | 9 +++++++++ pyproject.toml | 2 +- setup.cfg | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 03b4c58d..e84861d3 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,15 @@ This file contains a description of the major changes to the EESSI test suite. For more detailed information, please see the git log. +v0.3.1 (28 June 2024) +--------------------- + +This is a bugfix release of the EESSI test-suite + +It includes: + +- Correct required memory per node to MiB in ESPResSo test (#158) + v0.3.0 (27 June 2024) --------------------- diff --git a/pyproject.toml b/pyproject.toml index cfdefeeb..ecde89a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "eessi-testsuite" -version = "0.3.0" +version = "0.3.1" description = "Test suite for the EESSI software stack" readme = "README.md" license = {file = "LICENSE"} diff --git a/setup.cfg b/setup.cfg index 28c7fc13..35af40db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eessi-testsuite -version = 0.3.0 +version = 0.3.1 description = Test suite for the EESSI software stack long_description = file: README.md long_description_content_type = text/markdown From 8167eeea40e9ffbcf182193a536db596426b9ff1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 16:41:32 +0200 Subject: [PATCH 102/119] Change behavior on systems that have hyperthreading enabled for the assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) hook. Previous behaviour is that this launches one 1 task per hardware thread. We now change that to launch one task per physical core. A new COMPUTE_UNIT is introduced (COMPUTE_UNIT[HWTHREAD]) in case one wants to retain the previous behavior of launching one task per hardware thread. --- eessi/testsuite/constants.py | 2 + eessi/testsuite/hooks.py | 54 ++++++++++++++++--- .../testsuite/tests/apps/espresso/espresso.py | 4 ++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/eessi/testsuite/constants.py b/eessi/testsuite/constants.py index 19ad4a3d..0988e9be 100644 --- a/eessi/testsuite/constants.py +++ b/eessi/testsuite/constants.py @@ -4,6 +4,7 @@ AMD = 'AMD' CI = 'CI' +HWTHREAD = 'HWTHREAD' CPU = 'CPU' CPU_SOCKET = 'CPU_SOCKET' GPU = 'GPU' @@ -19,6 +20,7 @@ } COMPUTE_UNIT = { + HWTHREAD: 'hwthread', CPU: 'cpu', CPU_SOCKET: 'cpu_socket', GPU: 'gpu', diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index ab711955..14b61774 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -66,20 +66,18 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n Total task count is determined based on the number of nodes to be used in the test. Behaviour of this function is (usually) sensible for MPI tests. + WARNING: when using COMPUTE_UNIT[HWTHREAD] and invoking a hook for process binding, please verify that process binding happens correctly + Arguments: - test: the ReFrame test to which this hook should apply - compute_unit: a device as listed in eessi.testsuite.constants.COMPUTE_UNIT Examples: On a single node with 2 sockets, 64 cores and 128 hyperthreads: - - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) will launch 64 tasks with 1 thread - - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU_SOCKET]) will launch 2 tasks with 32 threads per task + - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[HWTHREAD]) will launch 128 tasks with 1 thread per task + - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) will launch 64 tasks with 2 threads per task + - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU_SOCKET]) will launch 2 tasks with 64 threads per task - Future work: - Currently, on a single node with 2 sockets, 64 cores and 128 hyperthreads, this - - assign_one_task_per_compute_unit(test, COMPUTE_UNIT[CPU], true) launches 128 tasks with 1 thread - - assign_one_task_per_compute_unit(test, COMPUTE_UNIT[CPU_SOCKET], true) launches 2 tasks with 64 threads per task - In the future, we'd like to add an arugment that disables spawning tasks for hyperthreads. """ if num_per != 1 and compute_unit in [COMPUTE_UNIT[GPU], COMPUTE_UNIT[CPU], COMPUTE_UNIT[CPU_SOCKET]]: raise NotImplementedError( @@ -106,6 +104,8 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n if compute_unit == COMPUTE_UNIT[GPU]: _assign_one_task_per_gpu(test) + elif compute_unit == COMPUTE_UNIT[HWTHREAD]: + _assign_one_task_per_hwthread(test) elif compute_unit == COMPUTE_UNIT[CPU]: _assign_one_task_per_cpu(test) elif compute_unit == COMPUTE_UNIT[CPU_SOCKET]: @@ -223,6 +223,41 @@ def _assign_one_task_per_cpu(test: rfm.RegressionTest): --setvar num_tasks_per_node= and/or --setvar num_cpus_per_task=. + Default resources requested: + - num_tasks_per_node = default_num_cpus_per_node + - num_cpus_per_task = default_num_cpus_per_node / num_tasks_per_node + """ + # neither num_tasks_per_node nor num_cpus_per_task are set + if not test.num_tasks_per_node and not test.num_cpus_per_task: + check_proc_attribute_defined(test, 'num_cpus_per_core') + test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core) + test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) + + # num_tasks_per_node is not set, but num_cpus_per_task is + elif not test.num_tasks_per_node: + test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.num_cpus_per_task) + + # num_cpus_per_task is not set, but num_tasks_per_node is + elif not test.num_cpus_per_task: + test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) + + else: + pass # both num_tasks_per_node and num_cpus_per_node are already set + + test.num_tasks = test.num_nodes * test.num_tasks_per_node + + log(f'num_tasks_per_node set to {test.num_tasks_per_node}') + log(f'num_cpus_per_task set to {test.num_cpus_per_task}') + log(f'num_tasks set to {test.num_tasks}') + + +def _assign_one_task_per_hwthread(test: rfm.RegressionTest): + """ + Sets num_tasks_per_node and num_cpus_per_task such that it will run one task per core, + unless specified with: + --setvar num_tasks_per_node= and/or + --setvar num_cpus_per_task=. + Default resources requested: - num_tasks_per_node = default_num_cpus_per_node - num_cpus_per_task = default_num_cpus_per_node / num_tasks_per_node @@ -508,6 +543,10 @@ def set_compact_process_binding(test: rfm.RegressionTest): This hook sets a binding policy for process binding. More specifically, it will bind each process to subsequent domains of test.num_cpus_per_task cores. + Arguments: + - test: the ReFrame test to which this hook should apply + + A few examples: - Pure MPI (test.num_cpus_per_task = 1) will result in binding 1 process to each core. this will happen in a compact way, i.e. rank 0 to core 0, rank 1 to core 1, etc @@ -522,6 +561,7 @@ def set_compact_process_binding(test: rfm.RegressionTest): # Check if hyperthreading is enabled. If so, divide the number of cpus per task by the number # of hw threads per core to get a physical core count + # TODO: check if this also leads to sensible binding when using COMPUTE_UNIT[HWTHREAD] check_proc_attribute_defined(test, 'num_cpus_per_core') num_cpus_per_core = test.current_partition.processor.num_cpus_per_core physical_cpus_per_task = int(test.num_cpus_per_task / num_cpus_per_core) diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index a87b22d1..5e47b231 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -65,6 +65,10 @@ def set_mem(self): mem_required_per_node = self.num_tasks_per_node * 0.9 hooks.req_memory_per_node(test=self, app_mem_req=mem_required_per_node * 1024) + @run_after('setup') + def set_binding(self): + hooks.set_compact_process_binding(self) + @deferrable def assert_completion(self): '''Check completion''' From 04a8997459ec2945ad98188e8acffbd8d546a0f8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 17:16:38 +0200 Subject: [PATCH 103/119] Fix linter issues --- eessi/testsuite/hooks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 14b61774..d71fd7ba 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -66,7 +66,8 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n Total task count is determined based on the number of nodes to be used in the test. Behaviour of this function is (usually) sensible for MPI tests. - WARNING: when using COMPUTE_UNIT[HWTHREAD] and invoking a hook for process binding, please verify that process binding happens correctly + WARNING: when using COMPUTE_UNIT[HWTHREAD] and invoking a hook for process binding, please verify that process + binding happens correctly. Arguments: - test: the ReFrame test to which this hook should apply @@ -230,7 +231,9 @@ def _assign_one_task_per_cpu(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cpus_per_core') - test.num_tasks_per_node = int(test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core) + test.num_tasks_per_node = int( + test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core + ) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) # num_tasks_per_node is not set, but num_cpus_per_task is From 70b100846e4f41274ee454517a50d307e807f856 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 17:43:44 +0200 Subject: [PATCH 104/119] Fix issue if num_cpus_per_node is set from the SCALES by set_tag_scale. If set there, we imply we want a single process on a single physical core. --- eessi/testsuite/hooks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index d71fd7ba..808d36f3 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -99,6 +99,16 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n ) _assign_default_num_cpus_per_node(test) + # If on + # - a hyperthreading system + # - num_cpus_per_node was set by the scale + # - compute_unit != COMPUTE_UNIT[HWTHREAD] + # double the default_num_cpus_per_node. In this scenario, if the scale asks for e.g. 1 num_cpus_per_node and + # the test doesn't state it wants to use hwthreads, we want to launch on two hyperthreads, i.e. one physical core + if SCALES[test.scale].get('num_cpus_per_node') is not None and compute_unit != COMPUTE_UNIT[HWTHREAD]: + check_proc_attribute_defined(test, 'num_cpus_per_core') + num_cpus_per_core = test.current_partition.processor.num_cpus_per_core + test.default_num_cpus_per_node = test.default_num_cpus_per_node * num_cpus_per_core if FEATURES[GPU] in test.current_partition.features: _assign_default_num_gpus_per_node(test) From 00c47952ad4151c72e6d05877f82c1dc74a8074c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 17:56:10 +0200 Subject: [PATCH 105/119] Always at least 1 task per node. I.e. even on a 2-core node, if we use 1/4 node, we should launch 1 task --- eessi/testsuite/hooks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 808d36f3..09f54c00 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -241,8 +241,9 @@ def _assign_one_task_per_cpu(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cpus_per_core') - test.num_tasks_per_node = int( - test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core + test.num_tasks_per_node = min( + int(test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core), + 1 ) test.num_cpus_per_task = int(test.default_num_cpus_per_node / test.num_tasks_per_node) From eba036ee17ec223a9701c7c669fde73c8deb6cff Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 17:58:52 +0200 Subject: [PATCH 106/119] Min was supposed to be Max --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 09f54c00..8688f2e7 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -241,7 +241,7 @@ def _assign_one_task_per_cpu(test: rfm.RegressionTest): # neither num_tasks_per_node nor num_cpus_per_task are set if not test.num_tasks_per_node and not test.num_cpus_per_task: check_proc_attribute_defined(test, 'num_cpus_per_core') - test.num_tasks_per_node = min( + test.num_tasks_per_node = max( int(test.default_num_cpus_per_node / test.current_partition.processor.num_cpus_per_core), 1 ) From 09b4bc5577f639cb1ad399b2e7f733a772d0a193 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 28 Jun 2024 18:23:31 +0200 Subject: [PATCH 107/119] Make more explicit how we only alter test.default_num_cpus_per_core on hyperthreading-enabled systems --- eessi/testsuite/hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 8688f2e7..413ccf79 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -108,7 +108,9 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n if SCALES[test.scale].get('num_cpus_per_node') is not None and compute_unit != COMPUTE_UNIT[HWTHREAD]: check_proc_attribute_defined(test, 'num_cpus_per_core') num_cpus_per_core = test.current_partition.processor.num_cpus_per_core - test.default_num_cpus_per_node = test.default_num_cpus_per_node * num_cpus_per_core + # On a hyperthreading system? + if num_cpus_per_core > 1: + test.default_num_cpus_per_node = test.default_num_cpus_per_node * num_cpus_per_core if FEATURES[GPU] in test.current_partition.features: _assign_default_num_gpus_per_node(test) From 79d792140e396b03a0d2565b0906bf010cb6c668 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 28 Jun 2024 18:27:01 +0200 Subject: [PATCH 108/119] include #160 in release notes for v0.3.1 --- RELEASE_NOTES | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index e84861d3..baddd1fa 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -9,6 +9,8 @@ This is a bugfix release of the EESSI test-suite It includes: - Correct required memory per node to MiB in ESPResSo test (#158) +- Change behavior for assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) on hyperthreading-enabled systems (#160) +- Use compat process binding in ESPResSo test (#160) v0.3.0 (27 June 2024) --------------------- From 9addf9a5cfc27f47dcef09125cc80275d343cc42 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 28 Jun 2024 18:35:34 +0200 Subject: [PATCH 109/119] fix typo --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index baddd1fa..c0730c52 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -10,7 +10,7 @@ It includes: - Correct required memory per node to MiB in ESPResSo test (#158) - Change behavior for assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU]) on hyperthreading-enabled systems (#160) -- Use compat process binding in ESPResSo test (#160) +- Use compact process binding in ESPResSo test (#160) v0.3.0 (27 June 2024) --------------------- From 3b022baa8226c56b5ab29465f09ea197c719712d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Jun 2024 12:19:07 +0200 Subject: [PATCH 110/119] bump version to v0.3.2 + update release notes --- CI/run_reframe.sh | 2 +- RELEASE_NOTES | 9 +++++++++ pyproject.toml | 2 +- setup.cfg | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CI/run_reframe.sh b/CI/run_reframe.sh index 57f209ca..1f439375 100755 --- a/CI/run_reframe.sh +++ b/CI/run_reframe.sh @@ -48,7 +48,7 @@ if [ -z "${EESSI_TESTSUITE_URL}" ]; then EESSI_TESTSUITE_URL='https://github.com/EESSI/test-suite.git' fi if [ -z "${EESSI_TESTSUITE_BRANCH}" ]; then - EESSI_TESTSUITE_BRANCH='v0.3.0' + EESSI_TESTSUITE_BRANCH='v0.3.2' fi if [ -z "${EESSI_CVMFS_REPO}" ]; then export EESSI_CVMFS_REPO=/cvmfs/software.eessi.io diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c0730c52..fa6ae01a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,15 @@ This file contains a description of the major changes to the EESSI test suite. For more detailed information, please see the git log. +v0.3.2 (29 June 2024) +--------------------- + +This is a bugfix release of the EESSI test-suite + +It includes: + +- Add config for Deucalion (#162) + v0.3.1 (28 June 2024) --------------------- diff --git a/pyproject.toml b/pyproject.toml index ecde89a4..445232af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "eessi-testsuite" -version = "0.3.1" +version = "0.3.2" description = "Test suite for the EESSI software stack" readme = "README.md" license = {file = "LICENSE"} diff --git a/setup.cfg b/setup.cfg index 35af40db..af7a3b6a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = eessi-testsuite -version = 0.3.1 +version = 0.3.2 description = Test suite for the EESSI software stack long_description = file: README.md long_description_content_type = text/markdown From 974cfd5dfb6452aaf531541fe0f4c6355859fcf3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Jun 2024 12:16:51 +0200 Subject: [PATCH 111/119] add config for Deucalion --- config/macc_deucalion.py | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 config/macc_deucalion.py diff --git a/config/macc_deucalion.py b/config/macc_deucalion.py new file mode 100644 index 00000000..7b9f1cb1 --- /dev/null +++ b/config/macc_deucalion.py @@ -0,0 +1,75 @@ +import os + +from eessi.testsuite.common_config import common_logging_config, common_general_config, common_eessi_init +from eessi.testsuite.constants import * # noqa: F403 + +# This config will write all staging, output and logging to subdirs under this prefix +# Override with RFM_PREFIX environment variable +reframe_prefix = os.path.join(os.environ['HOME'], 'reframe_runs') + +# This is an example configuration file +site_configuration = { + 'systems': [ + { + 'name': 'deucalion', + 'descr': 'Deucalion, a EuroHPC JU system', + 'modules_system': 'lmod', + 'hostnames': ['ln*', 'cn*', 'gn*'], + 'prefix': reframe_prefix, + 'partitions': [ + { + 'name': 'arm', + 'scheduler': 'slurm', + 'prepare_cmds': [ + 'wrap.sh << EOF', + # bypass CPU autodetection for now aarch64/a64fx, + # see https://github.com/EESSI/software-layer/pull/608 + 'export EESSI_SOFTWARE_SUBDIR_OVERRIDE=aarch64/a64fx', + 'source %s' % common_eessi_init(), + # Pass job environment variables like $PATH, etc., into job steps + 'export SLURM_EXPORT_ENV=HOME,PATH,LD_LIBRARY_PATH,PYTHONPATH', + ], + 'launcher': 'mpirun', + # Use --export=None to avoid that login environment is passed down to submitted jobs + 'access': ['-p normal-arm', '--export=None'], + 'environs': ['default'], + 'max_jobs': 120, + 'resources': [ + { + 'name': 'memory', + 'options': ['--mem={size}'], + } + ], + 'features': [ + FEATURES[CPU], + ] + list(SCALES.keys()), + 'extras': { + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + # NB: Deucalion's MaxMemPerNode is undefined. Experimentally I found you cannot submit with + # more than --mem=30000M + 'mem_per_node': 30000 # in MiB + }, + 'descr': 'CPU ARM A64FX partition, see https://docs.macc.fccn.pt/deucalion/#compute-nodes' + }, + ] + }, + ], + 'environments': [ + { + 'name': 'default', + 'cc': 'cc', + 'cxx': '', + 'ftn': '', + }, + ], + 'logging': common_logging_config(reframe_prefix), + 'general': [ + { + # Enable automatic detection of CPU architecture for each partition + # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information + 'remote_detect': True, + **common_general_config(reframe_prefix) + } + ], +} From 8298e6a190645752b70b60e4a086a511566e866e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 1 Jul 2024 12:24:35 +0200 Subject: [PATCH 112/119] Fix comments from Review Sam --- eessi/testsuite/hooks.py | 5 +++-- .../tests/apps/PyTorch/PyTorch_torchvision.py | 21 +++---------------- ..._get_free_socket.py => get_free_socket.py} | 0 3 files changed, 6 insertions(+), 20 deletions(-) rename eessi/testsuite/tests/apps/PyTorch/src/{python_get_free_socket.py => get_free_socket.py} (100%) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index a354cba8..fd82087f 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -58,7 +58,8 @@ def _assign_default_num_gpus_per_node(test: rfm.RegressionTest): def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, num_per: int = 1): """ - Assign one task per compute unit. + Assign one task per compute unit. More than 1 task per compute unit can be assigned with + num_per for compute units that support it. Automatically sets num_tasks, num_tasks_per_node, num_cpus_per_task, and num_gpus_per_node, based on the current scale and the current partition’s num_cpus, max_avail_gpus_per_node and num_nodes. For GPU tests, one task per GPU is set, and num_cpus_per_task is based on the ratio of CPU-cores/GPUs. @@ -80,7 +81,7 @@ def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, n - assign_tasks_per_compute_unit(test, COMPUTE_UNIT[CPU_SOCKET]) will launch 2 tasks with 64 threads per task """ - if num_per != 1 and compute_unit in [COMPUTE_UNIT[GPU], COMPUTE_UNIT[CPU], COMPUTE_UNIT[CPU_SOCKET]]: + if num_per != 1 and compute_unit not in [COMPUTE_UNIT[NODE]]: raise NotImplementedError( f'Non-default num_per {num_per} is not implemented for compute_unit {compute_unit}.') diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 575527cf..2235ac36 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -63,19 +63,18 @@ def apply_setup_hooks(self): if self.compute_device == DEVICE_TYPES[GPU]: hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[GPU]) else: - # Hybrid code, so launch 1 rank per socket. - # Probably, launching 1 task per NUMA domain is even better, but the current hook doesn't support it + # Hybrid code, for which launching one task per NUMA_NODE is typically the most efficient hooks.assign_tasks_per_compute_unit(test=self, compute_unit=COMPUTE_UNIT[NUMA_NODE]) # This is a hybrid test, binding is important for performance hooks.set_compact_process_binding(self) @run_after('setup') - def set_ddp_env_vars(self): + def set_ddp_options(self): # Set environment variables for PyTorch DDP if self.parallel_strategy == 'ddp': # Set additional options required by DDP - self.executable_opts += ["--master-port $(python python_get_free_socket.py)"] + self.executable_opts += ["--master-port $(python get_free_socket.py)"] self.executable_opts += ["--master-address $(hostname --fqdn)"] self.executable_opts += ["--world-size %s" % self.num_tasks] @@ -96,15 +95,6 @@ def pass_parallel_strategy(self): if self.num_tasks != 1: self.executable_opts += ['--use-%s' % self.parallel_strategy] - @run_after('setup') - def avoid_horovod_cpu_contention(self): - # Horovod had issues with CPU performance, see https://github.com/horovod/horovod/issues/2804 - # The root cause is Horovod having two threads with very high utilization, which interferes with - # the compute threads. It was fixed, but seems to be broken again in Horovod 0.28.1 - # The easiest workaround is to reduce the number of compute threads by 2 - if self.compute_device == DEVICE_TYPES[CPU] and self.parallel_strategy == 'horovod': - self.env_vars['OMP_NUM_THREADS'] = max(self.num_cpus_per_task - 2, 2) # Never go below 2 compute threads - @sanity_function def assert_num_ranks(self): '''Assert that the number of reported CPUs/GPUs used is correct''' @@ -140,8 +130,3 @@ def prepare_gpu_test(self): if self.precision == 'mixed': self.executable_opts += ['--use-amp'] - @run_after('init') - def skip_hvd_plus_amp(self): - '''Skip combination of horovod and AMP, it does not work see https://github.com/horovod/horovod/issues/1417''' - if self.parallel_strategy == 'horovod' and self.precision == 'mixed': - self.valid_systems = [INVALID_SYSTEM] diff --git a/eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py b/eessi/testsuite/tests/apps/PyTorch/src/get_free_socket.py similarity index 100% rename from eessi/testsuite/tests/apps/PyTorch/src/python_get_free_socket.py rename to eessi/testsuite/tests/apps/PyTorch/src/get_free_socket.py From af30b642af7d96d678eeaae0658a8a1e5b5e8f38 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 1 Jul 2024 12:48:28 +0200 Subject: [PATCH 113/119] Make linter happy --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 2235ac36..37630970 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -6,7 +6,7 @@ from reframe.core.builtins import parameter, variable, run_after, sanity_function, performance_function from eessi.testsuite import hooks -from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU, INVALID_SYSTEM +from eessi.testsuite.constants import SCALES, TAGS, DEVICE_TYPES, COMPUTE_UNIT, CPU, NUMA_NODE, GPU from eessi.testsuite.utils import find_modules @@ -129,4 +129,3 @@ def prepare_gpu_test(self): # Set precision if self.precision == 'mixed': self.executable_opts += ['--use-amp'] - From 7ddeedb11241f9d51ca0c40a069f359190570b17 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 1 Jul 2024 12:54:41 +0200 Subject: [PATCH 114/119] Remove training whitespace --- eessi/testsuite/hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index fd82087f..553e4d7f 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -59,7 +59,7 @@ def _assign_default_num_gpus_per_node(test: rfm.RegressionTest): def assign_tasks_per_compute_unit(test: rfm.RegressionTest, compute_unit: str, num_per: int = 1): """ Assign one task per compute unit. More than 1 task per compute unit can be assigned with - num_per for compute units that support it. + num_per for compute units that support it. Automatically sets num_tasks, num_tasks_per_node, num_cpus_per_task, and num_gpus_per_node, based on the current scale and the current partition’s num_cpus, max_avail_gpus_per_node and num_nodes. For GPU tests, one task per GPU is set, and num_cpus_per_task is based on the ratio of CPU-cores/GPUs. From d62443b10a19371507a3fc5d76c0abac1e2534d8 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 25 Jul 2024 16:32:37 +0200 Subject: [PATCH 115/119] Add set_omp_num_threads hook from https://github.com/EESSI/test-suite/pull/133 --- eessi/testsuite/hooks.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 553e4d7f..260a94b2 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -2,6 +2,7 @@ Hooks for adding tags, filtering and setting job resources in ReFrame tests """ import math +import os import shlex import warnings @@ -680,6 +681,15 @@ def set_compact_thread_binding(test: rfm.RegressionTest): log(f'Set environment variable KMP_AFFINITY to {test.env_vars["KMP_AFFINITY"]}') +def set_omp_num_threads(test: rfm.RegressionTest): + """ + Set default number of OpenMP threads equal to number of CPUs per task, + unless OMP_NUM_THREADS is already set + """ + test.env_vars['OMP_NUM_THREADS'] = os.getenv('OMP_NUM_THREADS', test.num_cpus_per_task) + log(f'Set environment variable OMP_NUM_THREADS to {test.env_vars["OMP_NUM_THREADS"]}') + + def _check_always_request_gpus(test: rfm.RegressionTest): """ Make sure we always request enough GPUs if required for the current GPU partition (cluster-specific policy) From 00fca31c7904b0e3918e5c5e1a8667567491f9ae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 25 Jul 2024 16:34:10 +0200 Subject: [PATCH 116/119] Call hook to set OMP_NUM_THREADS --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 37630970..f5922ca6 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -69,6 +69,9 @@ def apply_setup_hooks(self): # This is a hybrid test, binding is important for performance hooks.set_compact_process_binding(self) + # Set OMP_NUM_THREADS based on the number of cores per task + hooks.set_omp_num_threads(self) + @run_after('setup') def set_ddp_options(self): # Set environment variables for PyTorch DDP From a69e2d3e68be827f0afd5382793507c9cff53998 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 25 Jul 2024 16:49:38 +0200 Subject: [PATCH 117/119] Revert using the hook, it doesn't make sense to set OMP_NUM_THREADS conditionally, as this would be evaluated on the login node. That environment is irrelevant to the batch job --- eessi/testsuite/hooks.py | 10 ---------- .../tests/apps/PyTorch/PyTorch_torchvision.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/eessi/testsuite/hooks.py b/eessi/testsuite/hooks.py index 260a94b2..553e4d7f 100644 --- a/eessi/testsuite/hooks.py +++ b/eessi/testsuite/hooks.py @@ -2,7 +2,6 @@ Hooks for adding tags, filtering and setting job resources in ReFrame tests """ import math -import os import shlex import warnings @@ -681,15 +680,6 @@ def set_compact_thread_binding(test: rfm.RegressionTest): log(f'Set environment variable KMP_AFFINITY to {test.env_vars["KMP_AFFINITY"]}') -def set_omp_num_threads(test: rfm.RegressionTest): - """ - Set default number of OpenMP threads equal to number of CPUs per task, - unless OMP_NUM_THREADS is already set - """ - test.env_vars['OMP_NUM_THREADS'] = os.getenv('OMP_NUM_THREADS', test.num_cpus_per_task) - log(f'Set environment variable OMP_NUM_THREADS to {test.env_vars["OMP_NUM_THREADS"]}') - - def _check_always_request_gpus(test: rfm.RegressionTest): """ Make sure we always request enough GPUs if required for the current GPU partition (cluster-specific policy) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index f5922ca6..890be234 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -70,7 +70,7 @@ def apply_setup_hooks(self): hooks.set_compact_process_binding(self) # Set OMP_NUM_THREADS based on the number of cores per task - hooks.set_omp_num_threads(self) + test.env_vars["OMP_NUM_THREADS"] = self.num_cpus_per_task @run_after('setup') def set_ddp_options(self): From 4c5c3e7df0dd1d1ae714da15c557ec360beb6b45 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 25 Jul 2024 16:51:23 +0200 Subject: [PATCH 118/119] test is not defined, should be 'self' --- eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 890be234..13171143 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -70,7 +70,7 @@ def apply_setup_hooks(self): hooks.set_compact_process_binding(self) # Set OMP_NUM_THREADS based on the number of cores per task - test.env_vars["OMP_NUM_THREADS"] = self.num_cpus_per_task + self.env_vars["OMP_NUM_THREADS"] = self.num_cpus_per_task @run_after('setup') def set_ddp_options(self): From 93186d41a8244408fe0a9b1ac592847bed25298a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 25 Jul 2024 16:57:20 +0200 Subject: [PATCH 119/119] Update Snellius config with H100 nodes --- config/surf_snellius.py | 42 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/config/surf_snellius.py b/config/surf_snellius.py index 827ca32c..25545331 100644 --- a/config/surf_snellius.py +++ b/config/surf_snellius.py @@ -84,13 +84,12 @@ }, 'descr': 'AMD Genoa CPU partition with native EESSI stack' }, - { - 'name': 'gpu', + 'name': 'gpu_A100', 'scheduler': 'slurm', 'prepare_cmds': ['source %s' % common_eessi_init()], 'launcher': 'mpirun', - 'access': ['-p gpu', '--export=None'], + 'access': ['-p gpu_a100', '--export=None'], 'environs': ['default'], 'max_jobs': 60, 'devices': [ @@ -121,6 +120,43 @@ }, 'descr': 'Nvidia A100 GPU partition with native EESSI stack' }, + { + 'name': 'gpu_H100', + 'scheduler': 'slurm', + 'prepare_cmds': ['source %s' % common_eessi_init()], + 'launcher': 'mpirun', + 'access': ['-p gpu_h100', '--export=None'], + 'environs': ['default'], + 'max_jobs': 60, + 'devices': [ + { + 'type': DEVICE_TYPES[GPU], + 'num_devices': 4, + } + ], + 'resources': [ + { + 'name': '_rfm_gpu', + 'options': ['--gpus-per-node={num_gpus_per_node}'], + }, + { + 'name': 'memory', + 'options': ['--mem={size}'], + } + ], + 'features': [ + FEATURES[GPU], + FEATURES[ALWAYS_REQUEST_GPUS], + ] + valid_scales_snellius_gpu, + 'extras': { + GPU_VENDOR: GPU_VENDORS[NVIDIA], + # Make sure to round down, otherwise a job might ask for more mem than is available + # per node + 'mem_per_node': 737280 # in MiB + }, + 'descr': 'Nvidia H100 GPU partition with native EESSI stack' + }, + ] }, ],