CI JJOB Tests using CMake (NOAA-EMC#3214)

Adding CI tests at the JJOB level using CMake/ctest wrappers: These individual JJOB tests have four distinct phases: - **Setup:** Creates a EXPDIR/COMROOT just for the individual JJOB - **Stage:** Moves the specific files into the COMROOT that are needed to run the JJOB specified in `${HOMEgfs}/ci/ctest/cases/{CASE}_{JJOB}.yaml` - **Execute:** Run the JJOB in batch (batch card extracted from XML via Rocoto) - **Validate:** Check the outputs also specified in the above yaml configure file (currently stubbed) Resolves NOAA-EMC#3204 --------- Co-authored-by: Terry McGuinness <[email protected]> Co-authored-by: Rahul Mahajan <[email protected]>
AntonMFernando-NOAA · Jan 17, 2025 · 01eeb24 · 01eeb24
1 parent 57ce1b0
commit 01eeb24
Show file tree

Hide file tree

Showing 13 changed files with 423 additions and 0 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------- #
+# Global Workflow
+# ------------------------------------------------------------------------- #
+
+# Check for minimum cmake requirement
+cmake_minimum_required( VERSION 3.20 FATAL_ERROR )
+
+project(global_workflow VERSION 1.0.0)
+
+include(GNUInstallDirs)
+enable_testing()
+
+# Build type.
+if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
+  message(STATUS "Setting build type to 'Release' as none was specified.")
+  set(CMAKE_BUILD_TYPE
+      "Release"
+      CACHE STRING "Choose the type of build." FORCE)
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+                                               "MinSizeRel" "RelWithDebInfo")
+endif()
+
+# Build global-workflow source codes
+# add_subdirectory(sorc)
+
+# Setup tests
+add_subdirectory(ctests)
diff --git a/ci/platforms/config.hera b/ci/platforms/config.hera
@@ -2,6 +2,8 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+
+export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

diff --git a/ci/platforms/config.orion b/ci/platforms/config.orion
@@ -2,6 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

diff --git a/ctests/CMakeLists.txt b/ctests/CMakeLists.txt
@@ -0,0 +1,106 @@
+# ------------------------------------------------------------------------- #
+# CTests for Global Workflow
+# ------------------------------------------------------------------------- #
+# These ctests correspond to JJOBs (individual Rocoto jobs) that can be
+# run independently, each requiring its own YAML definition of inputs
+# and configurations. By integrating with Rocoto, these jobs can be
+# validated, staged, and executed as self-contained tests using
+# their own data and test parameters.
+# ------------------------------------------------------------------------- #
+
+# Function to set a variable from an environment variable or default value
+function(set_from_env_or_default VAR_NAME ENV_VAR DEFAULT_VALUE)
+  if (DEFINED ENV{${ENV_VAR}} AND NOT DEFINED ${VAR_NAME})
+    set(${VAR_NAME} $ENV{${ENV_VAR}} CACHE STRING "Set from environment variable ${ENV_VAR}")
+  elseif(NOT DEFINED ${VAR_NAME} AND NOT ${DEFAULT_VALUE} STREQUAL "")
+    set(${VAR_NAME} ${DEFAULT_VALUE} CACHE STRING "Default value for ${VAR_NAME}")
+  endif()
+endfunction()
+
+# Set HOMEgfs
+if (NOT DEFINED HOMEgfs)
+  set(HOMEgfs ${PROJECT_SOURCE_DIR})
+endif()
+
+# Set RUNTESTS
+set_from_env_or_default(RUNTESTS RUNTESTS "${CMAKE_CURRENT_BINARY_DIR}/RUNTESTS")
+
+# Set HPC_ACCOUNT
+set_from_env_or_default(HPC_ACCOUNT HPC_ACCOUNT " ")
+if (NOT DEFINED HPC_ACCOUNT)
+  message(WARNING "HPC_ACCOUNT must be set. CTests will not be created.")
+  return()
+endif()
+
+# Set ICSDIR_ROOT
+set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "")
+if (NOT DEFINED ICSDIR_ROOT)
+  message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.")
+  return()
+endif()
+
+# Set STAGED_TESTS_DIR
+set_from_env_or_default(STAGED_TESTS_DIR STAGED_TESTS_DIR "")
+if (NOT DEFINED STAGED_TESTS_DIR)
+  message(WARNING "STAGED_TESTS_DIR must be set. CTests will not be created.")
+  return()
+endif()
+
+message(STATUS "gw: global-workflow baselines will be used from: '${HOMEgfs}'")
+message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
+message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
+message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
+message(STATUS "gw: global-workflow tests will use staged data from:  '${STAGED_TESTS_DIR}'")
+
+# Prepare test scripts
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/validate.sh.in
+                ${CMAKE_CURRENT_BINARY_DIR}/scripts/validate.sh @ONLY)
+
+function(AddJJOBTest)
+
+  set(prefix     ARG)
+  set(novals     NOTRAPFPE NOVALGRIND)
+  set(singlevals CASE JOB TEST_DATE)
+  set(multivals  TEST_DEPENDS)
+
+  cmake_parse_arguments(${prefix}
+                       "${novals}" "${singlevals}" "${multivals}"
+                        ${ARGN})
+
+  set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
+  set(CASE_PATH ${HOMEgfs}/ci/cases/pr)
+  set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
+
+  add_test(NAME test_${TEST_NAME}_setup
+    COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_stage
+      COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_execute
+      COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}")
+
+  # TODO - This is a stub for the validation step
+  add_test(NAME test_${TEST_NAME}_validate
+      COMMAND ./validate.sh ${TEST_NAME} ${CASE_YAML}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
+endfunction()
+
+AddJJOBTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)
diff --git a/ctests/README.md b/ctests/README.md
@@ -0,0 +1,58 @@
+# CTest Framework for NOAA Global Workflow
+
+This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations.
+
+## Overview
+
+The CTest framework consists of the following scripts:
+- **setup.sh.in**: Prepares the environment and creates the experiment.
+- **stage.sh.in**: Stages the input files needed to run a JJOB.
+- **execute.sh.in**: Executes the JJOB and monitors its status.
+- **validate.sh.in**: (TODO) Validates the results of the JJOB.
+
+## Usage
+
+### CMake Configuration
+
+To configure the CTest framework using CMake, you need to provide several environment variables or default values. Here is an example of how to configure and build the project:
+
+```bash
+# Set environment variables (may also be include at command line with -D)
+export HPC_ACCOUNT="your_hpc_account"
+export ICSDIR_ROOT="/path/to/icsdir_root"
+export STAGED_TESTS_DIR="/path/to/staged_tests_dir"
+
+# Run CMake to configure the ctest framework
+cmake -S /path/to/HOMEgfs -B /path/to/build -DRUNTESTS=/path/to/runtests
+
+```
+
+### Running Tests with CTest
+
+Once the project is configured, you can run the tests using CTest. Here are some examples:
+
+#### Run All Tests
+
+```bash
+cd /path/to/build
+ctest
+```
+
+#### Run Tests for a Specific Case
+
+You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case:
+
+```bash
+cd /path/to/build
+ctest -L C48_ATM
+```
+
+To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows:
+```cmake
+AddJJOBTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)
+```
+Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml`
diff --git a/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml b/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml
@@ -0,0 +1,17 @@
+input_files:
+    mkdir:
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input"
+    copy:
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_ctrl.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile6.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile6.nc"]
diff --git a/ctests/scripts/execute.sh.in b/ctests/scripts/execute.sh.in
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -xe
+
+TEST_NAME=${1:?"Name of the test is required"}
+JOB=${2:?"Job name is required"}
+# TODO - adding idate by hand for now, need to get this from the test somehow 
+idate=$3
+
+#TODO - add rocotoboot_dryrun to repo some how
+rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
+CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
+cd "${CASEDIR}"
+rm -f ./*.db
+rm -f ./jobcard                                                                             
+
+yes | "${rocotoboot_dryrun}" -d "${TEST_NAME}.db" -w "${TEST_NAME}.xml" -v 10 -c "${idate}00" -t "${JOB}" 2> jobcard || true
+sed '/^{{\|^}}/d' < jobcard | sed '1d'  > "${TEST_NAME}.sub" || true
+
+#TODO - Generalize for batch system (hard coded to slurm)
+
+output=$(sbatch "${TEST_NAME}.sub")
+job_id=$(echo "${output}" | awk '{print $4}')
+echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}"
+
+# First loop: wait until job appears
+lack_of_job_count=0
+LACK_OF_JOB_LIMIT=5
+
+while true; do
+    job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1) || true
+    if [[ -n "${job_status}" ]]; then
+        echo "Job ${job_id} found in sacct."
+        break
+    fi
+    echo "Job ${job_id} not in sacct yet, attempt ${lack_of_job_count}/${LACK_OF_JOB_LIMIT}."
+    lack_of_job_count=$((lack_of_job_count + 1))
+    if [[ "${lack_of_job_count}" -ge "${LACK_OF_JOB_LIMIT}" ]]; then
+        echo "Job ${job_id} not found after ${lack_of_job_count} attempts. Exiting."
+        exit 1
+    fi
+    sleep 30
+done
+
+# Second loop: monitor job status until completion or failure
+timeout=0
+TIMEOUT=60
+while true; do
+    # Trim trailing spaces from job_status
+    job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | xargs) || true
+    if [[ "${job_status}" == "COMPLETED" ]]; then
+        echo "Job ${job_id} completed successfully."
+        break
+    elif [[ "${job_status}" =~ ^(FAILED|CANCELLED|TIMEOUT)$ ]]; then
+        echo "Job ${job_id} failed with status: ${job_status}."
+        exit 1
+    else
+        echo "Job ${job_id} is still running with status: ${job_status}."
+        sleep 60
+        timeout=$((timeout + 1))
+        if [[ "${timeout}" -gt "${TIMEOUT}" ]]; then
+            echo "Job ${job_id} has been running for more than ${TIMEOUT} minutes. Exiting."
+            exit 1
+        fi
+    fi
+done
diff --git a/ctests/scripts/setup.sh.in b/ctests/scripts/setup.sh.in
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+YAML_FILE=${2:?"Name of the CI yaml file for the test"}
+
+# CMake to fill these variables
+HOMEgfs="@PROJECT_SOURCE_DIR@"
+RUNTESTS="@RUNTESTS@"
+ICSDIR_ROOT="@ICSDIR_ROOT@"
+HPC_ACCOUNT="@HPC_ACCOUNT@"
+
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+set -x
+
+pslot="${TEST_NAME}" \
+RUNTESTS="${RUNTESTS}" \
+ICSDIR_ROOT="${ICSDIR_ROOT}" \
+HPC_ACCOUNT="${HPC_ACCOUNT}" \
+"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0
diff --git a/ctests/scripts/stage.py b/ctests/scripts/stage.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+"""
+stage.py
+
+This script is part of the ctest framework for testing Rocoto JJOBS that stages the
+input files needed to run a JJOB independently from other jobs in the workflow.
+The YAML file specified at the command line contains the paths to the staged input files
+and their corresponding directories under the COMROOT of the experiment for the JJOB.
+
+Usage:
+    stage.py -y <yaml_file> [-d <test_date>]
+
+Arguments:
+    -y, --yaml      Path to the YAML file describing the job test configuration (required)
+    -d, --test_date Test date in YYYYMMDDHH format (optional)
+
+Example:
+    ./stage.py -y /path/to/config.yaml -d 2021032312
+"""
+
+import os
+import datetime
+
+from argparse import ArgumentParser
+from pathlib import Path
+from wxflow import parse_j2yaml, FileHandler, Logger
+
+# Initialize logger with environment variable for logging level
+logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
+
+
+def parse_args():
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    argparse.Namespace
+        The parsed command line arguments, including:
+        - yaml: Path to the YAML file describing the job test configuration.
+        - test_date: Optional test date in YYYYMMDDHH format.
+    """
+    description = """Arguments for creating and updating error log files
+    """
+    parser = ArgumentParser(description=description)
+
+    # Add argument for YAML file path
+    parser.add_argument('-y', '--yaml', help='full path to yaml file describing the job test configuration', type=Path, required=True)
+    # Add optional argument for test date
+    parser.add_argument('-d', '--test_date', help='test date in YYYYMMDDHH format', type=str, required=False)
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    # Parse command line arguments
+    args = parse_args()
+
+    data = {}
+    if args.test_date:
+        # Parse test date from string to datetime object
+        data['TEST_DATE'] = datetime.datetime.strptime(args.test_date, '%Y%m%d%H')
+    # Parse YAML configuration file with optional data
+    case_cfg = parse_j2yaml(path=args.yaml, data=data)
+    # Synchronize input files as per the parsed configuration
+    FileHandler(case_cfg.input_files).sync()