diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..5044689f7e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------- # +# Global Workflow +# ------------------------------------------------------------------------- # + +# Check for minimum cmake requirement +cmake_minimum_required( VERSION 3.20 FATAL_ERROR ) + +project(global_workflow VERSION 1.0.0) + +include(GNUInstallDirs) +enable_testing() + +# Build type. +if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$") + message(STATUS "Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE + "Release" + CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") +endif() + +# Build global-workflow source codes +# add_subdirectory(sorc) + +# Setup tests +add_subdirectory(ctests) diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera index 6d3e43c820..09e2e28ddb 100644 --- a/ci/platforms/config.hera +++ b/ci/platforms/config.hera @@ -2,6 +2,8 @@ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR + +export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion index 5171373127..507068d4e7 100644 --- a/ci/platforms/config.orion +++ b/ci/platforms/config.orion @@ -2,6 +2,7 @@ export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR +export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR export HPC_ACCOUNT=nems export max_concurrent_cases=5 export max_concurrent_pr=4 diff --git a/ctests/CMakeLists.txt b/ctests/CMakeLists.txt new file mode 100644 index 0000000000..f8d928f456 --- /dev/null +++ b/ctests/CMakeLists.txt @@ -0,0 +1,106 @@ +# ------------------------------------------------------------------------- # +# CTests for Global Workflow +# ------------------------------------------------------------------------- # +# These ctests correspond to JJOBs (individual Rocoto jobs) that can be +# run independently, each requiring its own YAML definition of inputs +# and configurations. By integrating with Rocoto, these jobs can be +# validated, staged, and executed as self-contained tests using +# their own data and test parameters. +# ------------------------------------------------------------------------- # + +# Function to set a variable from an environment variable or default value +function(set_from_env_or_default VAR_NAME ENV_VAR DEFAULT_VALUE) + if (DEFINED ENV{${ENV_VAR}} AND NOT DEFINED ${VAR_NAME}) + set(${VAR_NAME} $ENV{${ENV_VAR}} CACHE STRING "Set from environment variable ${ENV_VAR}") + elseif(NOT DEFINED ${VAR_NAME} AND NOT ${DEFAULT_VALUE} STREQUAL "") + set(${VAR_NAME} ${DEFAULT_VALUE} CACHE STRING "Default value for ${VAR_NAME}") + endif() +endfunction() + +# Set HOMEgfs +if (NOT DEFINED HOMEgfs) + set(HOMEgfs ${PROJECT_SOURCE_DIR}) +endif() + +# Set RUNTESTS +set_from_env_or_default(RUNTESTS RUNTESTS "${CMAKE_CURRENT_BINARY_DIR}/RUNTESTS") + +# Set HPC_ACCOUNT +set_from_env_or_default(HPC_ACCOUNT HPC_ACCOUNT " ") +if (NOT DEFINED HPC_ACCOUNT) + message(WARNING "HPC_ACCOUNT must be set. CTests will not be created.") + return() +endif() + +# Set ICSDIR_ROOT +set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "") +if (NOT DEFINED ICSDIR_ROOT) + message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.") + return() +endif() + +# Set STAGED_TESTS_DIR +set_from_env_or_default(STAGED_TESTS_DIR STAGED_TESTS_DIR "") +if (NOT DEFINED STAGED_TESTS_DIR) + message(WARNING "STAGED_TESTS_DIR must be set. CTests will not be created.") + return() +endif() + +message(STATUS "gw: global-workflow baselines will be used from: '${HOMEgfs}'") +message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'") +message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'") +message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'") +message(STATUS "gw: global-workflow tests will use staged data from: '${STAGED_TESTS_DIR}'") + +# Prepare test scripts +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in + ${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in + ${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in + ${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/validate.sh.in + ${CMAKE_CURRENT_BINARY_DIR}/scripts/validate.sh @ONLY) + +function(AddJJOBTest) + + set(prefix ARG) + set(novals NOTRAPFPE NOVALGRIND) + set(singlevals CASE JOB TEST_DATE) + set(multivals TEST_DEPENDS) + + cmake_parse_arguments(${prefix} + "${novals}" "${singlevals}" "${multivals}" + ${ARGN}) + + set(TEST_NAME ${ARG_CASE}_${ARG_JOB}) + set(CASE_PATH ${HOMEgfs}/ci/cases/pr) + set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml) + + add_test(NAME test_${TEST_NAME}_setup + COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) + set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}") + + add_test(NAME test_${TEST_NAME}_stage + COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) + set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}") + + add_test(NAME test_${TEST_NAME}_execute + COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) + set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}") + + # TODO - This is a stub for the validation step + add_test(NAME test_${TEST_NAME}_validate + COMMAND ./validate.sh ${TEST_NAME} ${CASE_YAML} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) + set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}") +endfunction() + +AddJJOBTest( + CASE "C48_ATM" + JOB "gfs_fcst_seg0" + TEST_DATE "2021032312" +) diff --git a/ctests/README.md b/ctests/README.md new file mode 100644 index 0000000000..95a32cd952 --- /dev/null +++ b/ctests/README.md @@ -0,0 +1,58 @@ +# CTest Framework for NOAA Global Workflow + +This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations. + +## Overview + +The CTest framework consists of the following scripts: +- **setup.sh.in**: Prepares the environment and creates the experiment. +- **stage.sh.in**: Stages the input files needed to run a JJOB. +- **execute.sh.in**: Executes the JJOB and monitors its status. +- **validate.sh.in**: (TODO) Validates the results of the JJOB. + +## Usage + +### CMake Configuration + +To configure the CTest framework using CMake, you need to provide several environment variables or default values. Here is an example of how to configure and build the project: + +```bash +# Set environment variables (may also be include at command line with -D) +export HPC_ACCOUNT="your_hpc_account" +export ICSDIR_ROOT="/path/to/icsdir_root" +export STAGED_TESTS_DIR="/path/to/staged_tests_dir" + +# Run CMake to configure the ctest framework +cmake -S /path/to/HOMEgfs -B /path/to/build -DRUNTESTS=/path/to/runtests + +``` + +### Running Tests with CTest + +Once the project is configured, you can run the tests using CTest. Here are some examples: + +#### Run All Tests + +```bash +cd /path/to/build +ctest +``` + +#### Run Tests for a Specific Case + +You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case: + +```bash +cd /path/to/build +ctest -L C48_ATM +``` + +To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows: +```cmake +AddJJOBTest( + CASE "C48_ATM" + JOB "gfs_fcst_seg0" + TEST_DATE "2021032312" +) +``` +Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml` diff --git a/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml b/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml new file mode 100644 index 0000000000..ec0ce88ff1 --- /dev/null +++ b/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml @@ -0,0 +1,17 @@ +input_files: + mkdir: + - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input" + copy: + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_ctrl.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile1.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile2.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile3.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile4.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile5.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile6.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile1.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile2.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile3.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile4.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile5.nc"] + - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile6.nc"] diff --git a/ctests/scripts/execute.sh.in b/ctests/scripts/execute.sh.in new file mode 100755 index 0000000000..9cf3ef5917 --- /dev/null +++ b/ctests/scripts/execute.sh.in @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -xe + +TEST_NAME=${1:?"Name of the test is required"} +JOB=${2:?"Job name is required"} +# TODO - adding idate by hand for now, need to get this from the test somehow +idate=$3 + +#TODO - add rocotoboot_dryrun to repo some how +rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot +CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}" +cd "${CASEDIR}" +rm -f ./*.db +rm -f ./jobcard + +yes | "${rocotoboot_dryrun}" -d "${TEST_NAME}.db" -w "${TEST_NAME}.xml" -v 10 -c "${idate}00" -t "${JOB}" 2> jobcard || true +sed '/^{{\|^}}/d' < jobcard | sed '1d' > "${TEST_NAME}.sub" || true + +#TODO - Generalize for batch system (hard coded to slurm) + +output=$(sbatch "${TEST_NAME}.sub") +job_id=$(echo "${output}" | awk '{print $4}') +echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}" + +# First loop: wait until job appears +lack_of_job_count=0 +LACK_OF_JOB_LIMIT=5 + +while true; do + job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1) || true + if [[ -n "${job_status}" ]]; then + echo "Job ${job_id} found in sacct." + break + fi + echo "Job ${job_id} not in sacct yet, attempt ${lack_of_job_count}/${LACK_OF_JOB_LIMIT}." + lack_of_job_count=$((lack_of_job_count + 1)) + if [[ "${lack_of_job_count}" -ge "${LACK_OF_JOB_LIMIT}" ]]; then + echo "Job ${job_id} not found after ${lack_of_job_count} attempts. Exiting." + exit 1 + fi + sleep 30 +done + +# Second loop: monitor job status until completion or failure +timeout=0 +TIMEOUT=60 +while true; do + # Trim trailing spaces from job_status + job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | xargs) || true + if [[ "${job_status}" == "COMPLETED" ]]; then + echo "Job ${job_id} completed successfully." + break + elif [[ "${job_status}" =~ ^(FAILED|CANCELLED|TIMEOUT)$ ]]; then + echo "Job ${job_id} failed with status: ${job_status}." + exit 1 + else + echo "Job ${job_id} is still running with status: ${job_status}." + sleep 60 + timeout=$((timeout + 1)) + if [[ "${timeout}" -gt "${TIMEOUT}" ]]; then + echo "Job ${job_id} has been running for more than ${TIMEOUT} minutes. Exiting." + exit 1 + fi + fi +done diff --git a/ctests/scripts/setup.sh.in b/ctests/scripts/setup.sh.in new file mode 100755 index 0000000000..6c4a772b65 --- /dev/null +++ b/ctests/scripts/setup.sh.in @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -ux + +TEST_NAME=${1:?"Name of the test is required"} +YAML_FILE=${2:?"Name of the CI yaml file for the test"} + +# CMake to fill these variables +HOMEgfs="@PROJECT_SOURCE_DIR@" +RUNTESTS="@RUNTESTS@" +ICSDIR_ROOT="@ICSDIR_ROOT@" +HPC_ACCOUNT="@HPC_ACCOUNT@" + +set +x +source "${HOMEgfs}/workflow/gw_setup.sh" +set -x + +pslot="${TEST_NAME}" \ +RUNTESTS="${RUNTESTS}" \ +ICSDIR_ROOT="${ICSDIR_ROOT}" \ +HPC_ACCOUNT="${HPC_ACCOUNT}" \ +"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite +rc=$? +if [[ "${rc}" -ne 0 ]]; then + set +x + echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'" + set -x + exit "${rc}" +fi + +exit 0 diff --git a/ctests/scripts/stage.py b/ctests/scripts/stage.py new file mode 100755 index 0000000000..b8a77a120d --- /dev/null +++ b/ctests/scripts/stage.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +""" +stage.py + +This script is part of the ctest framework for testing Rocoto JJOBS that stages the +input files needed to run a JJOB independently from other jobs in the workflow. +The YAML file specified at the command line contains the paths to the staged input files +and their corresponding directories under the COMROOT of the experiment for the JJOB. + +Usage: + stage.py -y [-d ] + +Arguments: + -y, --yaml Path to the YAML file describing the job test configuration (required) + -d, --test_date Test date in YYYYMMDDHH format (optional) + +Example: + ./stage.py -y /path/to/config.yaml -d 2021032312 +""" + +import os +import datetime + +from argparse import ArgumentParser +from pathlib import Path +from wxflow import parse_j2yaml, FileHandler, Logger + +# Initialize logger with environment variable for logging level +logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) + + +def parse_args(): + """ + Parse command line arguments. + + Returns + ------- + argparse.Namespace + The parsed command line arguments, including: + - yaml: Path to the YAML file describing the job test configuration. + - test_date: Optional test date in YYYYMMDDHH format. + """ + description = """Arguments for creating and updating error log files + """ + parser = ArgumentParser(description=description) + + # Add argument for YAML file path + parser.add_argument('-y', '--yaml', help='full path to yaml file describing the job test configuration', type=Path, required=True) + # Add optional argument for test date + parser.add_argument('-d', '--test_date', help='test date in YYYYMMDDHH format', type=str, required=False) + return parser.parse_args() + + +if __name__ == '__main__': + + # Parse command line arguments + args = parse_args() + + data = {} + if args.test_date: + # Parse test date from string to datetime object + data['TEST_DATE'] = datetime.datetime.strptime(args.test_date, '%Y%m%d%H') + # Parse YAML configuration file with optional data + case_cfg = parse_j2yaml(path=args.yaml, data=data) + # Synchronize input files as per the parsed configuration + FileHandler(case_cfg.input_files).sync() diff --git a/ctests/scripts/stage.sh.in b/ctests/scripts/stage.sh.in new file mode 100755 index 0000000000..9ced3d8f4e --- /dev/null +++ b/ctests/scripts/stage.sh.in @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -ux + +TEST_NAME=${1:?"Name of the test is required"} +TEST_DATE=${2:?"idate of the test is required"} + +# CMake to fill these variables +STAGED_TESTS_DIR="@STAGED_TESTS_DIR@" +RUNTESTS="@RUNTESTS@" +HOMEgfs="@PROJECT_SOURCE_DIR@" + +# Load the runtime environment for this script (needs wxflow and its dependencies) +set +x +source "${HOMEgfs}/workflow/gw_setup.sh" +rc=$? +[[ "${rc}" -ne 0 ]] && exit "${status}" +set -x +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" +export PYTHONPATH + +INPUTS_YAML="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml" + +TEST_NAME="${TEST_NAME}" \ +RUNTESTS="${RUNTESTS}" \ +STAGED_TESTS_DIR="${STAGED_TESTS_DIR}" \ +"${HOMEgfs}/ctests/scripts/stage.py" --yaml "${INPUTS_YAML}" --test_date "${TEST_DATE}" +rc=$? +if [[ "${rc}" -ne 0 ]]; then + set +x + echo "Failed to stage inputs for '${TEST_NAME}' with '${INPUTS_YAML}'" + set -x + exit "${rc}" +fi + +exit 0 diff --git a/ctests/scripts/validate.sh.in b/ctests/scripts/validate.sh.in new file mode 100755 index 0000000000..0277699956 --- /dev/null +++ b/ctests/scripts/validate.sh.in @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ux + +TEST_NAME=${1:?"Name of the test is required"} +YAML_FILE=${2:?"Name of the CI yaml file for validating the test"} + +echo "validating '${TEST_NAME}' with yaml file '${YAML_FILE}'" + +exit 0 \ No newline at end of file diff --git a/modulefiles/module_gwsetup.hercules.lua b/modulefiles/module_gwsetup.hercules.lua index 9d845fb71d..e7735e4aa1 100644 --- a/modulefiles/module_gwsetup.hercules.lua +++ b/modulefiles/module_gwsetup.hercules.lua @@ -9,6 +9,7 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local python_ver=os.getenv("python_ver") or "3.11.6" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("python", python_ver)) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index b8e2fc8a9f..5ffebc31a1 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -10,11 +10,13 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-st local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local python_ver=os.getenv("python_ver") or "3.11.6" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") +load(pathJoin("cmake", cmake_ver)) whatis("Description: GFS run setup environment")