sub_mc21_version

#!/bin/bash

NAME=mc21
upper_NAME=`echo ${NAME} | tr '[a-z]' '[A-Z]'`
VERSION=6.01.01
EXE_NAME="mc21HostStart mc21.0${VERSION}.INL"
MPI_CMD=mpirun
EXTRA_MODULE_NAME1=use.exp_ctl
EXTRA_MODULE_NAME2=
MODULE_NAME=mc21/$VERSION
ND_LIBRARY_FULLPATH=${ND_LIBRARY_FULLPATH:-"/apps/exp_ctl/mc21/5.00.01/ND_LIBRARY_ENDF70_300K"}
#
#export MC21_TMPDIR=${MC21_TMPDIR:="$PWD"}
#
source /hpc-common/software/sub_scripts/common_sub_fns.sh
#source /home/garvct/my_sw_development/pbs_job_submit/common_sub_fns.sh
#


function usage ()
{
     echo
     echo "sub_mc21_${VERSION}"
     echo      "usage: sub_mc21_${VERSION} <-i inputfile> <-w HH:MM:SS> <-P project> [<-n mpiprocs>] [<-q queue>] [<-N mpipernode>] [<-j PBS_job_name>] [-f ND_LIBRARY_FULLPATH] [<-m PBS_memory>] [<-J pbs job_array_range>] [<-F job_array_filename>] [<-W depends_on_jobid>] [<-s scratch,ramdisk,localdisk>] [<-t cpu_type>] [-c] [<-D>] [<-g>] [<-h>]"
     echo      "Submits an ${NAME} version ${VERSION} job to the cluster via PBS Pro."
     echo      " Where:"
     echo      " -i inputfile  : Selects the Input file.( Default: none)"
     echo      " -w HH:MM:SS   : Selects the PBS Job Walltime.( Default: none)"
     echo      " -P project    : Selects the project this job is associated with.( Default: none)"
     echo      " -n mpiprocs   : Selects the number of parallel MPI processes to use.( Default: 1)"
     echo      " -q queue      : Selects the PBS queue to use.( Default: general)"
     echo      " -N mpipernode : Selects the maximum number of parallel mpi processes to use in each node.( Default: ${MPI_PER_NODE})"
     echo      " -j jobname    : Selects the name of PBS job.( Default: input file name)"
     echo      " -f ND_LIBRARY_FULLPATH : Selects the Nuclear Data to use.( Default: $ND_LIBRARY_FULLPATH)"
     echo      " -m PBS_memory : Selects the PBS Memory per Node.( Default: PBS defaults)"
     echo      " -J PBS_job_array_range : Selects the PBS job array range (e.g 1-10)( Default: None)"
     echo      " -F PBS_job_array_filename : Selects the PBS job array Filename. Each line of file contains"
     echo      "                             the full path to the input file. (Default: None)"
     echo      " -W depend_on_jobid : This job runs after jobid is complete. (Default: none)"
     echo      " -s scratch,ramdisk,localdisk : Job runs in scratch(panasas parallel filesystem),"
     echo      "                                ramdisk(memory where available) or localdisk (locally attached disk where available). (Default: Run from current location)"
     echo      " -t cpu_type   : Applicable on falcon cluster only, requests the cpu type (\"haswell\" requests haswell processors, \"broadwell\" requests broadwell processors (default "haswell")"
     echo      " -c            : Check Input (CheckInput)( Default: Do not check input)"
     echo      " -D            : Selects the debug option (performance basic system checks, tracks memory usage, process placement etc)( Default: off)"
     echo      " -g            : Generate PBS script only, but do not submit (Default: Generate and submit)"
     echo      " -h            : Selects help.( Default: none)"
     echo
     echo "NOTE: To add extra PBS options (e.g -M email_address -m be), use the environmental variable PBS_EXTRA_ARGS"
     echo       'e.g. export PBS_EXTRA_ARGS="-M cormac.garvey@inl.gov -m be"'
     echo       " To add extra MC21 command line arguments use the MC21_EXTRA_ARGS environmental variables."
     echo
}

while getopts "i:w:P:n:q:N:j:f:m:J:F:W:s:t:cDghX" opt; do
  case $opt in
    i)
      INPUT=$OPTARG
      ;;
    w)
      WALLTIME=$OPTARG
      ;;
    P)
      PROJECT=$OPTARG
      ;;
    n)
      MPIPROCS=$OPTARG
      ;;
    q)
      QUEUE=$OPTARG
      ;;
    N)
      MPI_PER_NODE=$OPTARG
      MPI_PER_NODE_SET=$OPTARG
      ;;
    j)
      JOBNAME=$OPTARG
      ;;
    f)
      ND_LIBRARY_FULLPATH=$OPTARG
      ;;
    m)
      PBS_MEM=$OPTARG
      ;;
    J)
      PBS_JOB_ARRAY_RANGE=$OPTARG
      ;;
    F)
      PBS_JOB_ARRAY_FILENAME=$OPTARG
      ;;
    W)
      DEPENDS_ON_JOBID=$OPTARG
      ;;
    s)
      SCRATCH_TYPE=$OPTARG
      ;;
    t)
      CPU_TYPE=$OPTARG
      ;;
    c)
      CHECK_INPUT=1
      ;;
    D)
      DEBUG_JOB=1
      ;;
    g)
      SUBMIT_JOB=0
      ;;
    h)
      HELP=1
      usage
      exit 0
      ;;
    X)
      exit 0
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      usage
      exit
      ;;
  esac
done

check_command_args
set_default_jobname
calc_chunk_new

if [ -n "$SCRATCH_TYPE" ]
then
   get_scratch_dir $SCRATCH_TYPE
fi

if [ -z "$PBS_JOB_ARRAY_FILENAME" ]
then
   ln -sf $ND_LIBRARY_FULLPATH ND_LIBRARY
fi

#echo "2 NNODES = $NNODES RPROCS = $RPROCS"

echo  "#!/bin/bash"                                                                                          > $JOBNAME.pbs
pbs_select $JOBNAME.pbs
pbs_set_defaults $JOBNAME.pbs
pbs_depends_on $JOBNAME.pbs
pbs_jobarray $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs
echo  "cat \$PBS_NODEFILE"                                                                                  >> $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs

load_modules $JOBNAME.pbs

echo  "#"                                                                                                   >> $JOBNAME.pbs
echo  "echo \"The following PBS extra args (PBS_EXTRA_ARGS) will be used: $PBS_EXTRA_ARGS\""                >> $JOBNAME.pbs
echo  "echo \"The following MC21 extra args (MC21_EXTRA_ARGS) will be used: $MC21_EXTRA_ARGS\""             >> $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs
echo  "cd \$PBS_O_WORKDIR"                                                                                  >> $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs

debug_job $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs
setup_run_in_scratch $JOBNAME.pbs $INPUT
echo  "#"                                                                                                   >> $JOBNAME.pbs
setup_jobarray $JOBNAME.pbs
echo  "#"                                                                                                   >> $JOBNAME.pbs

if [ -n "$PBS_JOB_ARRAY_RANGE" ]
then
   EXE_ARGS="-input \$input_name"
else
   EXE_ARGS="-input $INPUT"
fi
if [ -n "$CHECK_INPUT" ]
then
   EXE_ARGS=$EXE_ARGS" -checkInput"
fi

if [ -z "$DEBUG_JOB" ]
then
   echo "${MPI_CMD} ${MPI_ARGS} ${EXE_NAME} ${MC21_EXTRA_ARGS} ${EXE_ARGS}"                                 >> $JOBNAME.pbs
else
   echo "job_tracker.py \"${MPI_CMD} ${MPI_ARGS} ${EXE_NAME} ${MC21_EXTRA_ARGS} ${EXE_ARGS}\""              >> $JOBNAME.pbs
fi

echo  "#"                                                                                                   >> $JOBNAME.pbs
finish_jobarray_and_scratch $JOBNAME.pbs

if [ $SUBMIT_JOB -eq 1 ]
then
   qsub $JOBNAME.pbs
fi