Skip to content

Commit

Permalink
Simplify validation run scripts (arbor-sim#56)
Browse files Browse the repository at this point in the history 
Move common functionality to run-validation.sh.
Implements arbor-sim#53.

* Make validation run script responsible for configuring output directory, printing test status, and capturing test stdout/stderr.
* Pass cache refresh request via environment variable `ns_cache_refresh`.
* Use exit codes to return test failure status.
* Print progress, platform information to stdout in run-validation.sh.
* Set up terminal colours with `tput`, and test for tty.
  • Loading branch information
Sam Yates authored Mar 28, 2019
1 parent b92accb commit 99f71de
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 115 deletions.
89 changes: 71 additions & 18 deletions run-validation.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ for modeldir in "$ns_base_path/validation/"*; do
fi
done

unset ns_refresh_cache
unset ns_cache_refresh
unset ns_validation_output_format

while [ -n "$1" ]; do
Expand Down Expand Up @@ -102,7 +102,7 @@ while [ -n "$1" ]; do
models="$models $1"
;;
-r | --refresh )
ns_refresh_cache="-r"
ns_cache_refresh=1
;;
-* | --* )
argerror "unknown option '$1'"
Expand All @@ -124,31 +124,48 @@ ns_prefix=$(full_path "$ns_prefix")

source "$ns_base_path/scripts/environment.sh"
default_environment
export ns_validation_output_format

export ns_cache_refresh

# Add common/bin scripts to our path (used for git-repo-hash, pathsub).
export PATH="$ns_base_path/common/bin${PATH:+:}$PATH"

# TODO: this has to go into the configuration environment setup scripts
export ARB_NUM_THREADS=$[ $ns_threads_per_core * $ns_cores_per_socket ]

msg "---- Platform ----"
msg "platform: $ns_system"
msg "cores per socket: $ns_cores_per_socket"
msg "threads per core: $ns_threads_per_core"
msg "threads: $ARB_NUM_THREADS"
msg "sockets: $ns_sockets"
msg "mpi: $ns_with_mpi"
info "---- Platform ----"
info "platform: $ns_system"
info "cores per socket: $ns_cores_per_socket"
info "threads per core: $ns_threads_per_core"
info "threads: $ARB_NUM_THREADS"
info "sockets: $ns_sockets"
info "mpi: $ns_with_mpi"
echo

msg "---- Validation ----"
echo
info "---- Validation ----"

# Colour highlight shortcuts:
red=${tcol[hi_red]}
green=${tcol[hi_green]}
cyan=${tcol[hi_cyan]}
nc=${tcol[reset]}

# Grab git repo hash and install timestamp for substitution in output directories below.
# TODO: Move this sort of thing into a config/common_env.sh script at install time?
repo_hash=$(git-repo-hash ${ns_base_path})
repo_hash_short=$(git-repo-hash --short ${ns_base_path})
ns_timestamp=$(< "$ns_build_path/timestamp")
ns_sysname=$(< "$ns_build_path/sysname")

for sim in $sims; do
echo
sim_env="$ns_prefix/config/env_$sim.sh"
if [ ! -f "$sim_env" ]; then
echo "Simulator $sim has not been locally installed, skipping."
info "Simulator $sim has not been locally installed, skipping."
continue
fi

echo "Running validation for $sim:"
info "Running validation for $sim:"

for model in $models; do
param=""
Expand All @@ -160,18 +177,54 @@ for sim in $sims; do

model_path="$ns_base_path/validation/$basemodel"
if [ ! -x "$model_path/run" ]; then
echo "Missing run file for model $basemodel, skipping."
info "Missing run file for model $basemodel, skipping."
continue
fi

if [ ! -r "$model_path/$param.param" ]; then
echo "Missing parameter file $param.param for model $basemodel, skipping."
info "Missing parameter file $param.param for model $basemodel, skipping."
continue
fi

outdir=$(pathsub --base="$ns_validation_output" \
T="$ns_timestamp" S="$ns_sysname" H="$repo_hash" h="$repo_hash_short" \
s="$sim" m="$basemodel" p="$param" \
-- \
"${ns_validation_output_format:-%s/%m/%p}")

mkdir -p "$outdir" || exit_on_err "run-validation.sh: cannot create directory '$outdir'"

model_status="$outdir/status"
test_id="$sim $basemodel/$param"

# Run script exit codes:
# 0 => success: test passed.
# 96 => failure: test run but validation failed.
# 97 => missing: no implementation for given simulator.
# other => error: execution error.

(
source "$sim_env";
"$model_path/run" $ns_refresh_cache "$sim" "$param"
)
"$model_path/run" "$outdir" "$sim" "$param"
) > "$outdir/run.out" 2> "$outdir/run.err"

case $? in
0 )
echo "$green[PASS]$nc $test_id"
echo pass > "$model_status"
;;
96 )
echo "$red[FAIL]$nc $test_id"
echo fail > "$model_status"
;;
97 )
echo "$cyan[MISSING]$nc $test_id"
echo missing > "$model_status"
;;
* )
echo "$red[ERROR]$nc $test_id"
echo error > "$model_status"
;;
esac
done
done
77 changes: 12 additions & 65 deletions scripts/model_common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,55 +5,43 @@
# Requires PATH, PYTHONPATH and LD_LIBRARY_PATH to be already configured
# for installed and common binaries, scripts and libraries.
#
# The following nsuite working paths are also required:
# ns_validation_output
# ns_cache_path
# The following nsuite variables are used:
# ns_cache_path (required)
# ns_cache_refresh (optionally set)

function die {
echo "$@" >&2; exit 1
}

function exit_model_fail { exit 96; }
function exit_model_missing { exit 97; }

# Sets model_name, model_sim and model_param from arguments, provided in
# this order.
#
# Creates if required output dir and cache dir.
#
# Sets variables model_output_dir, model_cache_dir, model_param_data
# and model_status_path.
# Computes and sets variables model_cache_dir, model_param_data.
# Creates cache directory if not present.

function model_setup {
if [ -z "$ns_validation_output" -o -z "$ns_cache_path" ]; then
if [ -z "$ns_cache_path" ]; then
echo "error: missing required ns_ path variables"
exit 1
fi

model_name="$1"

model_refresh=""
if [ "$2" = "-r" ]; then model_refresh="-r"; shift; fi

model_sim="$2"
model_param="$3"

model_cache_dir="$ns_cache_path/$model_name"
mkdir -p "$model_cache_dir" || die "$model_name: cannot create directory '$model_cache_dir'"

local -a fmtkeys
fmtkeys=("T=$ns_timestamp" "S=$ns_sysname" "s=$model_sim" "m=$model_name" "p=$model_param")
fmtkeys+=("H=$(git-repo-hash)" "h=$(git-repo-hash --short)")

model_output_dir=$(pathsub --base="$ns_validation_output" "${fmtkeys[@]}" -- "${ns_validation_output_format:-%s/%m/%p}")

mkdir -p "$model_output_dir" || die "$model_name: cannot create directory '$model_output_dir'"

model_status_path="$model_output_dir/status"

[ -r "${model_param}.param" ] || die "$model_name: unable to read parameter data '${pset}.param'"
model_param_data=$(< ${model_param}.param)
}

# Print path to file if in CWD, or else relative to cache dir.
# Return non-zero if not in cache dir either.
# Return non-zero if not in cache dir, or if "$ns_cache_refresh" is
# set to a non-empty string.

function model_find_cacheable {
file="$1"
Expand All @@ -63,47 +51,6 @@ function model_find_cacheable {
else
local cached="$model_cache_dir/$file"
echo "$cached"
return $([ -e "$cached" ])
fi
}

# Report pass/fail status to stdout based on argument,
# zero => pass; non-zero => fail.

function model_notify_pass_fail {
local white=$'\033[1;37m'
local green=$'\033[1;92m'
local light_red=$'\033[1;31m'
local nc=$'\033[0m'

if [ "$1" -eq 0 ]; then
echo "${green}[PASS]${nc} $model_sim $model_name/$model_param"
else
echo "${light_red}[FAIL]${nc} $model_sim $model_name/$model_param"
return $([ -z "$ns_cache_refresh" -a -e "$cached" ])
fi
}

# Attempt to run given simulator-specific script, redirecting
# stdout and stderr to files in $model_output_dir.
#
# Report if error or if implementation is missing.

function model_try_run {
local light_red=$'\033[1;31m'
local magenta=$'\033[1;35m'
local nc=$'\033[0m'

impl="$1"
shift

if [ ! -x "./$impl" ]; then
echo "${magenta}[UNIMPLEMENTED]${nc} $model_sim $model_name/$model_param"
exit 0
fi

"./$impl" "${@}" > "$model_output_dir/run.out" 2> "$model_output_dir/run.err" || {\
echo "${light_red}[ERROR]${nc} $model_sim $model_name/$model_param"
exit 0
}
}

65 changes: 56 additions & 9 deletions scripts/util.sh
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,76 @@
init_tcol() {
# Set global associative array 'tcol' with entries for the eight
# commonly supported colours plus entries for other highlight modes.
#
# Leave it empty if we're not attached to a tty.

unset tcol
declare -gA tcol

if [[ -t 1 ]]; then
local -a colname=(black red green yellow blue magenta cyan white)
for i in ${!colname[@]}; do tcol[${colname[$i]}]="$(tput setaf $i)"; done

# If we have 16+ colours, set high intensity versions of colours to these,
# otherwise reuse the standard 8.

if [[ $(tput colors) -ge 16 ]]; then
for i in ${!colname[@]}; do
let j=8+i
tcol[hi_${colname[$i]}]="$(tput setaf $j)";
done
else
for i in ${!colname[@]}; do tcol[hi_${colname[$i]}]="${tcol[${colname[$i]}]}"; done
fi

# Other entries:
tcol[reset]=$(tput sgr0) # reset all attributes
tcol[bold]=$(tput bold) # bold
tcol[ul]=$(tput smul) # underline
tcol[noul]=$(tput rmul) # no underline
tcol[so]=$(tput smso) # standout
tcol[noso]=$(tput rmso) # no standout
fi
}

# Always initialize colours on sourcing of util.sh
init_tcol

# Print a message to stderr.
# Output to stderr to help determine where in build script an error occurred.
msg() {
local white='\033[1;37m'
local light_cyan='\033[1;36m'
local nc='\033[0m'
local white="${tcol[hi_white]}"
local light_cyan="${tcol[hi_cyan]}"
local nc="${tcol[reset]}"

>&2 printf "${light_cyan}== ${nc} ${white}$*${nc}\n"
}

err() {
local white='\033[1;37m'
local light_red='\033[1;31m'
local nc='\033[0m'
local white="${tcol[hi_white]}"
local light_red="${tcol[hi_red]}"
local nc="${tcol[reset]}"

>&2 printf "${light_red}== ERROR${nc} ${white}$*${nc}\n"
}

dbg() {
local white='\033[1;37m'
local green='\033[1;92m'
local nc='\033[0m'
local white="${tcol[hi_white]}"
local green="${tcol[hi_green]}"
local nc="${tcol[reset]}"

>&2 printf "${green}==== ${nc} ${white}$*${nc}\n"
}

# Print a message to stdout following msg() formatting.
info() {
local white="${tcol[hi_white]}"
local light_cyan="${tcol[hi_cyan]}"
local nc="${tcol[reset]}"

printf "${light_cyan}== ${nc} ${white}$*${nc}\n"
}

exit_on_error() {
err "$*"
exit 1
Expand Down
Loading

0 comments on commit 99f71de

Please sign in to comment.