diff --git a/run-validation.sh b/run-validation.sh index 8d0fd9d0b7..3193123596 100755 --- a/run-validation.sh +++ b/run-validation.sh @@ -68,7 +68,7 @@ for modeldir in "$ns_base_path/validation/"*; do fi done -unset ns_refresh_cache +unset ns_cache_refresh unset ns_validation_output_format while [ -n "$1" ]; do @@ -102,7 +102,7 @@ while [ -n "$1" ]; do models="$models $1" ;; -r | --refresh ) - ns_refresh_cache="-r" + ns_cache_refresh=1 ;; -* | --* ) argerror "unknown option '$1'" @@ -124,31 +124,48 @@ ns_prefix=$(full_path "$ns_prefix") source "$ns_base_path/scripts/environment.sh" default_environment -export ns_validation_output_format + +export ns_cache_refresh + +# Add common/bin scripts to our path (used for git-repo-hash, pathsub). +export PATH="$ns_base_path/common/bin${PATH:+:}$PATH" # TODO: this has to go into the configuration environment setup scripts export ARB_NUM_THREADS=$[ $ns_threads_per_core * $ns_cores_per_socket ] -msg "---- Platform ----" -msg "platform: $ns_system" -msg "cores per socket: $ns_cores_per_socket" -msg "threads per core: $ns_threads_per_core" -msg "threads: $ARB_NUM_THREADS" -msg "sockets: $ns_sockets" -msg "mpi: $ns_with_mpi" +info "---- Platform ----" +info "platform: $ns_system" +info "cores per socket: $ns_cores_per_socket" +info "threads per core: $ns_threads_per_core" +info "threads: $ARB_NUM_THREADS" +info "sockets: $ns_sockets" +info "mpi: $ns_with_mpi" echo -msg "---- Validation ----" -echo +info "---- Validation ----" + +# Colour highlight shortcuts: +red=${tcol[hi_red]} +green=${tcol[hi_green]} +cyan=${tcol[hi_cyan]} +nc=${tcol[reset]} + +# Grab git repo hash and install timestamp for substitution in output directories below. +# TODO: Move this sort of thing into a config/common_env.sh script at install time? +repo_hash=$(git-repo-hash ${ns_base_path}) +repo_hash_short=$(git-repo-hash --short ${ns_base_path}) +ns_timestamp=$(< "$ns_build_path/timestamp") +ns_sysname=$(< "$ns_build_path/sysname") for sim in $sims; do + echo sim_env="$ns_prefix/config/env_$sim.sh" if [ ! -f "$sim_env" ]; then - echo "Simulator $sim has not been locally installed, skipping." + info "Simulator $sim has not been locally installed, skipping." continue fi - echo "Running validation for $sim:" + info "Running validation for $sim:" for model in $models; do param="" @@ -160,18 +177,54 @@ for sim in $sims; do model_path="$ns_base_path/validation/$basemodel" if [ ! -x "$model_path/run" ]; then - echo "Missing run file for model $basemodel, skipping." + info "Missing run file for model $basemodel, skipping." continue fi if [ ! -r "$model_path/$param.param" ]; then - echo "Missing parameter file $param.param for model $basemodel, skipping." + info "Missing parameter file $param.param for model $basemodel, skipping." continue fi + outdir=$(pathsub --base="$ns_validation_output" \ + T="$ns_timestamp" S="$ns_sysname" H="$repo_hash" h="$repo_hash_short" \ + s="$sim" m="$basemodel" p="$param" \ + -- \ + "${ns_validation_output_format:-%s/%m/%p}") + + mkdir -p "$outdir" || exit_on_err "run-validation.sh: cannot create directory '$outdir'" + + model_status="$outdir/status" + test_id="$sim $basemodel/$param" + + # Run script exit codes: + # 0 => success: test passed. + # 96 => failure: test run but validation failed. + # 97 => missing: no implementation for given simulator. + # other => error: execution error. + ( source "$sim_env"; - "$model_path/run" $ns_refresh_cache "$sim" "$param" - ) + "$model_path/run" "$outdir" "$sim" "$param" + ) > "$outdir/run.out" 2> "$outdir/run.err" + + case $? in + 0 ) + echo "$green[PASS]$nc $test_id" + echo pass > "$model_status" + ;; + 96 ) + echo "$red[FAIL]$nc $test_id" + echo fail > "$model_status" + ;; + 97 ) + echo "$cyan[MISSING]$nc $test_id" + echo missing > "$model_status" + ;; + * ) + echo "$red[ERROR]$nc $test_id" + echo error > "$model_status" + ;; + esac done done diff --git a/scripts/model_common.sh b/scripts/model_common.sh index cdd081e8d0..1ce751b481 100755 --- a/scripts/model_common.sh +++ b/scripts/model_common.sh @@ -5,55 +5,43 @@ # Requires PATH, PYTHONPATH and LD_LIBRARY_PATH to be already configured # for installed and common binaries, scripts and libraries. # -# The following nsuite working paths are also required: -# ns_validation_output -# ns_cache_path +# The following nsuite variables are used: +# ns_cache_path (required) +# ns_cache_refresh (optionally set) function die { echo "$@" >&2; exit 1 } +function exit_model_fail { exit 96; } +function exit_model_missing { exit 97; } + # Sets model_name, model_sim and model_param from arguments, provided in # this order. # -# Creates if required output dir and cache dir. -# -# Sets variables model_output_dir, model_cache_dir, model_param_data -# and model_status_path. +# Computes and sets variables model_cache_dir, model_param_data. +# Creates cache directory if not present. function model_setup { - if [ -z "$ns_validation_output" -o -z "$ns_cache_path" ]; then + if [ -z "$ns_cache_path" ]; then echo "error: missing required ns_ path variables" exit 1 fi model_name="$1" - - model_refresh="" - if [ "$2" = "-r" ]; then model_refresh="-r"; shift; fi - model_sim="$2" model_param="$3" model_cache_dir="$ns_cache_path/$model_name" mkdir -p "$model_cache_dir" || die "$model_name: cannot create directory '$model_cache_dir'" - local -a fmtkeys - fmtkeys=("T=$ns_timestamp" "S=$ns_sysname" "s=$model_sim" "m=$model_name" "p=$model_param") - fmtkeys+=("H=$(git-repo-hash)" "h=$(git-repo-hash --short)") - - model_output_dir=$(pathsub --base="$ns_validation_output" "${fmtkeys[@]}" -- "${ns_validation_output_format:-%s/%m/%p}") - - mkdir -p "$model_output_dir" || die "$model_name: cannot create directory '$model_output_dir'" - - model_status_path="$model_output_dir/status" - [ -r "${model_param}.param" ] || die "$model_name: unable to read parameter data '${pset}.param'" model_param_data=$(< ${model_param}.param) } # Print path to file if in CWD, or else relative to cache dir. -# Return non-zero if not in cache dir either. +# Return non-zero if not in cache dir, or if "$ns_cache_refresh" is +# set to a non-empty string. function model_find_cacheable { file="$1" @@ -63,47 +51,6 @@ function model_find_cacheable { else local cached="$model_cache_dir/$file" echo "$cached" - return $([ -e "$cached" ]) - fi -} - -# Report pass/fail status to stdout based on argument, -# zero => pass; non-zero => fail. - -function model_notify_pass_fail { - local white=$'\033[1;37m' - local green=$'\033[1;92m' - local light_red=$'\033[1;31m' - local nc=$'\033[0m' - - if [ "$1" -eq 0 ]; then - echo "${green}[PASS]${nc} $model_sim $model_name/$model_param" - else - echo "${light_red}[FAIL]${nc} $model_sim $model_name/$model_param" + return $([ -z "$ns_cache_refresh" -a -e "$cached" ]) fi } - -# Attempt to run given simulator-specific script, redirecting -# stdout and stderr to files in $model_output_dir. -# -# Report if error or if implementation is missing. - -function model_try_run { - local light_red=$'\033[1;31m' - local magenta=$'\033[1;35m' - local nc=$'\033[0m' - - impl="$1" - shift - - if [ ! -x "./$impl" ]; then - echo "${magenta}[UNIMPLEMENTED]${nc} $model_sim $model_name/$model_param" - exit 0 - fi - - "./$impl" "${@}" > "$model_output_dir/run.out" 2> "$model_output_dir/run.err" || {\ - echo "${light_red}[ERROR]${nc} $model_sim $model_name/$model_param" - exit 0 - } -} - diff --git a/scripts/util.sh b/scripts/util.sh index 4e44dd2ea9..7316df37a7 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -1,29 +1,76 @@ +init_tcol() { + # Set global associative array 'tcol' with entries for the eight + # commonly supported colours plus entries for other highlight modes. + # + # Leave it empty if we're not attached to a tty. + + unset tcol + declare -gA tcol + + if [[ -t 1 ]]; then + local -a colname=(black red green yellow blue magenta cyan white) + for i in ${!colname[@]}; do tcol[${colname[$i]}]="$(tput setaf $i)"; done + + # If we have 16+ colours, set high intensity versions of colours to these, + # otherwise reuse the standard 8. + + if [[ $(tput colors) -ge 16 ]]; then + for i in ${!colname[@]}; do + let j=8+i + tcol[hi_${colname[$i]}]="$(tput setaf $j)"; + done + else + for i in ${!colname[@]}; do tcol[hi_${colname[$i]}]="${tcol[${colname[$i]}]}"; done + fi + + # Other entries: + tcol[reset]=$(tput sgr0) # reset all attributes + tcol[bold]=$(tput bold) # bold + tcol[ul]=$(tput smul) # underline + tcol[noul]=$(tput rmul) # no underline + tcol[so]=$(tput smso) # standout + tcol[noso]=$(tput rmso) # no standout + fi +} + +# Always initialize colours on sourcing of util.sh +init_tcol + # Print a message to stderr. # Output to stderr to help determine where in build script an error occurred. msg() { - local white='\033[1;37m' - local light_cyan='\033[1;36m' - local nc='\033[0m' + local white="${tcol[hi_white]}" + local light_cyan="${tcol[hi_cyan]}" + local nc="${tcol[reset]}" >&2 printf "${light_cyan}== ${nc} ${white}$*${nc}\n" } err() { - local white='\033[1;37m' - local light_red='\033[1;31m' - local nc='\033[0m' + local white="${tcol[hi_white]}" + local light_red="${tcol[hi_red]}" + local nc="${tcol[reset]}" >&2 printf "${light_red}== ERROR${nc} ${white}$*${nc}\n" } dbg() { - local white='\033[1;37m' - local green='\033[1;92m' - local nc='\033[0m' + local white="${tcol[hi_white]}" + local green="${tcol[hi_green]}" + local nc="${tcol[reset]}" >&2 printf "${green}==== ${nc} ${white}$*${nc}\n" } +# Print a message to stdout following msg() formatting. +info() { + local white="${tcol[hi_white]}" + local light_cyan="${tcol[hi_cyan]}" + local nc="${tcol[reset]}" + + printf "${light_cyan}== ${nc} ${white}$*${nc}\n" +} + exit_on_error() { err "$*" exit 1 diff --git a/validation/rc-exp2syn-spike/run b/validation/rc-exp2syn-spike/run index 3bbb5cb8df..a9dd095223 100755 --- a/validation/rc-exp2syn-spike/run +++ b/validation/rc-exp2syn-spike/run @@ -1,11 +1,18 @@ #!/usr/bin/env bash +# Invocation: run outputdir simname paramsetname # Change to script directory and attempt to find nsuite base directory. + unset CDPATH cd "${BASH_SOURCE[0]%/*}" [ -n "$ns_base_path" ] || ns_base_path="$(cd ../..; pwd)" -# Set up model paths and model_XXX variables. +# Set up model paths, model_XXX variables, and functions +# exit_model_fail, exit_model_missing. + +outdir=$1 +shift + source "$ns_base_path/scripts/model_common.sh" model_setup rc-exp2syn-spike "$@" @@ -18,21 +25,24 @@ model_setup rc-exp2syn-spike "$@" # For CoreNEURON, voltage traces will be omitted; the pass/fail # test only looks at spike time differences. -# Run sim-specific implementation with parameter data. -outfile="$model_output_dir/run.nc" -model_try_run "run-$model_sim" "$outfile" $model_param_data +impl="./run-$model_sim" +[ -x "$impl" ] || exit_model_missing + +outfile="$outdir/run.nc" +"$impl" "$outfile" $model_param_data || exit 1 # Generate reference data if required. -reffile=$(model_find_cacheable "ref-${model_name}-${model_param}.nc") -if [ $? -ne 0 -o -n "$model_refresh" ]; then - ./generate-rc-exp2syn-spike "$reffile" $model_param_data -fi + +reffile=$(model_find_cacheable "ref-${model_name}-${model_param}.nc") || \ + ./generate-rc-exp2syn-spike "$reffile" $model_param_data || exit 1 # Run comparison. -deltafile="$model_output_dir/delta.nc" -comparex "$outfile" --warn --ref "$reffile" --var spike -o "$deltafile" + +deltafile="$outdir/delta.nc" +comparex "$outfile" --warn --ref "$reffile" --var spike -o "$deltafile" || exit 1 # Get threshold from parameter data 'max_error' + max_error= for param in $model_param_data; do if [[ "$param" =~ ^'max_error='(.*) ]]; then @@ -44,5 +54,4 @@ if [ -z "$max_error" ]; then exit 1 fi -thresholdx "$deltafile" -e "spike.abserr<$max_error" > "$model_status_path" -model_notify_pass_fail $? +thresholdx "$deltafile" -e "spike.abserr<$max_error" || exit_model_fail diff --git a/validation/rc-expsyn/run b/validation/rc-expsyn/run index c3fc3ad5e1..24aeb786e2 100755 --- a/validation/rc-expsyn/run +++ b/validation/rc-expsyn/run @@ -1,27 +1,37 @@ #!/usr/bin/env bash +# Invocation: run outputdir simname paramsetname # Change to script directory and attempt to find nsuite base directory. + unset CDPATH cd "${BASH_SOURCE[0]%/*}" [ -n "$ns_base_path" ] || ns_base_path="$(cd ../..; pwd)" -# Set up model paths and model_XXX variables. +# Set up model paths, model_XXX variables, and functions +# exit_model_fail, exit_model_missing. + +outdir=$1 +shift + source "$ns_base_path/scripts/model_common.sh" model_setup rc-expsyn "$@" # Run sim-specific implementation with parameter data. -outfile="$model_output_dir/run.nc" -model_try_run "run-$model_sim" "$outfile" $model_param_data + +impl="./run-$model_sim" +[ -x "$impl" ] || exit_model_missing + +outfile="$outdir/run.nc" +"$impl" "$outfile" $model_param_data || exit 1 # Generate reference data if required. -reffile=$(model_find_cacheable "ref-${model_name}-${model_param}.nc") -if [ $? -ne 0 -o -n "$model_refresh" ]; then - ./generate-rc-expsyn "$reffile" $model_param_data -fi + +reffile=$(model_find_cacheable "ref-${model_name}-${model_param}.nc") || \ + ./generate-rc-expsyn "$reffile" $model_param_data || exit 1 # Run comparison. -deltafile="$model_output_dir/delta.nc" -comparex "$outfile" --warn --ref "$reffile" --interpolate time -o "$deltafile" -thresholdx "$deltafile" -e "voltage.relerr.lb<0.01" > "$model_status_path" -model_notify_pass_fail $? +deltafile="$outdir/delta.nc" +comparex "$outfile" --warn --ref "$reffile" --interpolate time -o "$deltafile" || exit 1 + +thresholdx "$deltafile" -e "voltage.relerr.lb<0.01" || exit_model_fail