farmoutRandomSeedJobs

#!/bin/bash
#
# Setup:

#
# Usage:
#       farmoutRandomSeedJobs <jobName> <nEvents> <nEventsPerJob> <CMSSW Version> <config file>
#
#  Example: farmoutCmsRunJobs blah 1000000 100 ~/CMSSW_1_2_0 /path/to/my.cfg
#
# The config file may refer to the following macros, which are automatically
# inserted by this script:
#
# $randomNumber
# $randomNumberN where N is any unique number
# $nEventsPerJob
# $outputFileName

# Initialize default settings:
FARMOUT_HOME=${FARMOUT_HOME:-$(dirname $(realpath $0))}

# for storing output
FARMOUT_USER=${FARMOUT_USER:-${USER}}
SRM_SERVER=srm://cmssrm2.hep.wisc.edu:8443/srm/v2/server?SFN=/hdfs
PNFS_HOME=/store/user/${FARMOUT_USER}
CMS_DASHBOARD_LOCAL_CE=cmsgrid02.hep.wisc.edu

SRM_HOME=${SRM_SERVER}${PNFS_HOME}

# We need wisc AFS for accing user CMSSW release area unless in no-shared-fs mode
SITE_REQUIREMENTS='TARGET.Arch == "X86_64" && (MY.RequiresSharedFS=!=true || TARGET.HasAFS_OSG) && (TARGET.OSG_major =!= undefined || TARGET.IS_GLIDEIN=?=true) && IsSlowSlot=!=true'

##
## cms.cern.ch depends on the cms.hep.wisc.edu CVMFS respository for NOW (02-19-2013)
## SITECONF is the reason for the dependency

if echo "$PATH" | egrep -q "(/cvmfs/cms.hep.wisc.edu)|(/cvmfs/cms.cern.ch)" ; then
  # require cvmfs to be available and at least as up to date as the local cvmfs

  SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && (TARGET.HasParrotCVMFS=?=true || (TARGET.UWCMS_CVMFS_Exists  && TARGET.CMS_CVMFS_Exists"

  local_cvmfs_revision=`attr -q -g revision /cvmfs/cms.hep.wisc.edu`
  if [ "$local_cvmfs_revision" != "" ]; then
    SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && TARGET.UWCMS_CVMFS_Revision >= ${local_cvmfs_revision}"
  fi

  local_cvmfs_revision=`attr -q -g revision /cvmfs/cms.cern.ch`
  if [ "$local_cvmfs_revision" != "" ]; then
    SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && TARGET.CMS_CVMFS_Revision >= ${local_cvmfs_revision}"
  fi

  SITE_REQUIREMENTS="${SITE_REQUIREMENTS} ))"
fi

DISK_REQUIREMENTS=2000
MEMORY_REQUIREMENTS=900
VSIZE_LIMIT=3000
HOLD_IMAGE_SIZE_FACTOR=4.0
HOLD_DISK_USAGE_FACTOR=10.0
MIN_PROXY_HOURS=24
OUTPUT_FILES_PER_SUBDIR=0

# We want explicit references to "app_rw" to be replaced with "app"
# so the interactive osg app volume is not pounded by the jobs.
SITE_ENV_MATCH_REPLACE_1='s|/afs/hep.wisc.edu/osg/app_[^/]*/|/afs/hep.wisc.edu/osg/app/|g'

# special exit status to force job to leave the queue
FAIL_JOB=42

for scratch_dir in /data /scratch /tmp; do
  if [ -d $scratch_dir ] && [ -w $scratch_dir ]; then
    break
  fi
done
SUBMIT_HOME=${scratch_dir}/${FARMOUT_USER}

SHARED_LOGS=${scratch_dir}/farmout_logs
mkdir -p -m a+rwx $SHARED_LOGS
SHARED_LOGS=$SHARED_LOGS/${FARMOUT_USER}
mkdir -p $SHARED_LOGS

logerror() {
  echo 2>&1 "$@"
}

die() {
  if [ $# -gt 0 ]; then
    logerror
    logerror "$@"
  fi
  exit 1
}

outputFileExists() {
  fname=$1

  #Strip off srm://hostname:8443 to get raw path.
  local_fname=${fname#srm://*:8443}

  #Strip off '/blah/blah?SFN='
  local_fname=${local_fname#*SFN=}

  if [ -f "$local_fname" ]; then
    return 0
  fi
  return 1
}

realpath() {
  if ! [ -a "$1" ]; then
    echo "$1"
  fi
  readlink -f $1
}

checkSharedFS() {
  local d=$1
  if echo "$d" | grep -q ^/afs/; then
    if ! (fs la "$d" | grep -q 'condor-hosts rl' || \
          fs la "$d" | grep -q 'system:anyuser rl'); then
       echo
       echo "WARNING: You are using --shared-fs, but $d"
       echo "may not be readable from the worker nodes.  Ensure that this directory"
       echo "and all sub-directories are readable without an AFS token.  Also ensure"
       echo "That parent directories are listable without an AFS token.  We recommend"
       echo "using the condor-hosts group to achieve this.  Example:"
       echo
       d=$(realpath $CMSSW_HOME/..)
       h=$(realpath ~)
       while [ "$d" != "/" ] && [ "$d" != "" ] && [ "$d" != "$h" ]; do
           if ! (fs la "$d" | grep -q 'condor-hosts r*l' || \
                fs la "$d" | grep -q 'system:anyuser r*l'); then
               echo fs setacl -dir $d -acl condor-hosts l
           fi
           d=$(realpath $d/..)
       done
       echo "find $CMSSW_HOME -type d -exec fs setacl -dir '{}' -acl condor-hosts rl \;"
       echo
    fi
  fi
}

check_proxy() {
  hours=$1
  proxy=$2
  if ! [ -f "$proxy" ]; then
    logerror
    logerror "NOTE: No grid proxy found.  (Expecting to find it here: $proxy.)"
    return 1
  fi

  #Issue a warning if less than this many seconds remain:
  min_proxy_lifetime=$((3600*$hours))

  seconds_left="`voms-proxy-info --timeleft --file=$proxy 2>/dev/null`"

  if [ "$seconds_left" = "" ]; then
    echo "WARNING: cannot find time remaining for grid proxy."
    voms-proxy-info -all -path $proxy
    return 0
  fi
  if [ "$seconds_left" -lt "$min_proxy_lifetime" ]; then
    logerror
    logerror "NOTE: grid proxy is about to expire:"
    logerror "voms-proxy-info"
    voms-proxy-info --file=$proxy
    return 1
  fi

}

packUserCode() {
  local release_area="$1"
  local tgz="$2"

  local saved_wd="$(pwd)"
  cd "$release_area" || die "Failed to access $release_area"

  # Include src/*/python directories, because python/ contains
  # symlinks to to it that must remain symlinks, due to code that
  # relies on its location within the src tree.
  # Also include src/*/data, because these may be needed.
  local src_files="$(find src -type d -name python -o -name data)"

  tar czf "$tgz" bin lib python $src_files || die "Failed to make $tgz"

  cd "$saved_wd" || die "Failed to access $saved_wd"
}

PrintUsage() {
  echo "USAGE: farmoutRandomSeedJobs [options] <jobName> <events> <events-per-job> <CMSSW Path> <config files> [extra-cmsRun-args]"
  echo ""
  echo "OPTIONS:"
  echo "  --output-dir=${SRM_HOME}/<jobName>"
  echo "               (--output-dir=. copies files back to submit dir)"
  echo "  --submit-dir=${SUBMIT_HOME}/<jobName>"
  echo "  --no-submit"
  echo "  --skip-existing-output  (do not create job if output file exists)"
  echo "  --skip-existing-jobs    (do not create job if already created)"
  echo "  --memory-requirements=$MEMORY_REQUIREMENTS (megabytes)"
  echo "  --vsize-limit=$VSIZE_LIMIT (megabytes)"
  echo "  --disk-requirements=$DISK_REQUIREMENTS  (megabytes)"
  echo "  --site-requirements=$SITE_REQUIREMENTS"
  echo "  --save-failed-datafiles  (save root file from failed cmsRun job)"
  echo "                           (in <output-dir>-cmsRun-failed)"
  echo "  --express-queue-only     (run in the express queue only)"
  echo "  --extra-inputs=file1,file2,...  (e.g. parameter files)"
  echo "  --accounting-group=cms_name  (username to use for fair-sharing)"
  echo "  --requires-whole-machine (job should reserve all slots on machine)"
  echo "  --output-files-per-subdir=N (0 for infinite; $OUTPUT_FILES_PER_SUBDIR default)"
  echo "  --pre-hook=<exe>            (call <exe> before the job runs)"
  echo "  --post-hook=<exe>           (call <exe> after the job runs)"
  echo "  --no-shared-fs              (the default: send analysis binaries to execute machine)"
  echo "  --shared-fs                 (rely on CMSSW project area being on a shared fs (e.g. AFS))"
  echo "  --use-osg                   (allow jobs to run opportunistically on OSG)"
  echo "  --use-only-osg              (only run jobs opportunistically on OSG)"
  echo ""
  exit 2
}

OPTS=`getopt -o "h" -l "help,output-dir:,submit-dir:,no-submit,skip-existing-output,skip-existing-jobs,disk-requirements:,memory-requirements:,save-failed-datafiles,site-requirements:,quick-test,express-queue-only,extra-inputs:,accounting-group:,requires-whole-machine,output-files-per-subdir:,vsize-limit:,pre-hook:,post-hook:,no-shared-fs,shared-fs,use-osg,use-only-osg" -- "$@"`
if [ $? -ne 0 ]; then PrintUsage; fi

eval set -- "$OPTS"

ACCOUNTING_GROUP=
NO_SUBMIT=
OUTPUT_DIR=
SUBMIT_DIR=
SKIP_EXISTING_OUTPUT=
SKIP_EXISTING_JOBS=
SAVE_FAILED_DATAFILES=
CFG_EXTENSION=
QUICK_TEST=
EXTRA_INPUTS=
REQUIRES_WHOLE_MACHINE=
FARMOUT_HOOK_PRERUN=
FARMOUT_HOOK_POSTRUN=
NO_SHARED_FS=1
USE_OSG=0

while [ ! -z "$1" ]
do
  case "$1" in
    -h) PrintUsage;;
    --help) PrintUsage;;
    --no-submit) NO_SUBMIT=1;;
    --output-dir) shift; OUTPUT_DIR=$1;;
    --submit-dir) shift; SUBMIT_DIR=$1;;
    --skip-existing-output) SKIP_EXISTING_OUTPUT=1;;
    --skip-existing-jobs) SKIP_EXISTING_JOBS=1;;
    --disk-requirements) shift; DISK_REQUIREMENTS=$1;;
    --memory-requirements) shift; MEMORY_REQUIREMENTS=$1;;
    --vsize-limit) shift; VSIZE_LIMIT=$1;;
    --save-failed-datafiles) SAVE_FAILED_DATAFILES=1;;
    --site-requirements) shift; SITE_REQUIREMENTS="$1";;
    --quick-test) QUICK_TEST=1;;
    --express-queue-only) QUICK_TEST=1;;
    --extra-inputs) shift; EXTRA_INPUTS="$1";;
    --accounting-group) shift; ACCOUNTING_GROUP="+AccountingGroup=\"$1\"";;
    --requires-whole-machine) REQUIRES_WHOLE_MACHINE=1;;
    --output-files-per-subdir) shift; OUTPUT_FILES_PER_SUBDIR=$1;;
    --pre-hook) shift; FARMOUT_HOOK_PRERUN="$1";;
    --post-hook) shift; FARMOUT_HOOK_POSTRUN="$1";;
    --no-shared-fs) NO_SHARED_FS=1;;
    --shared-fs) NO_SHARED_FS=0;;
    --use-osg) NO_SHARED_FS=1; USE_OSG=1;;
    --use-only-osg) NO_SHARED_FS=1; USE_OSG=1; SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && TARGET.IS_GLIDEIN";;
    --) shift; break;;
    *) die "Unexpected option $1";;
  esac
  shift
done

if [ "$#" -lt 5 ]; then PrintUsage; fi

# Check for some required utilities
for exe in scramv1 condor_submit cmsRun.sh readlink voms-proxy-info; do
  if ! which $exe >& /dev/null; then
    die "Cannot find $exe in PATH.  Your environment is not correctly set up."
  fi
done

# Additional command-line arguments

jobName=$1
declare -i nEvents=$2
declare -i nEventsPerJob=$3
CMSSW_HOME=`realpath $4`
configTemplate=`realpath $5`

shift 5
chainedConfigs=""
while [ "$1" != "" ]; do
   if [ "${1/*=*/is_cmsRunOption}" = "is_cmsRunOption" ]; then
     break
   fi
   cfg=`realpath $1`
   chainedConfigs="$chainedConfigs $cfg"
   shift
done

# get extra arguments in condor arg syntax
EXTRA_CMSRUN_ARGS=""
while [ "$1" != "" ]; do
  escaped_arg=$(echo $1 | sed "s|'|''|g; s|\"|\"\"|g")
  shift
  EXTRA_CMSRUN_ARGS="$EXTRA_CMSRUN_ARGS '$escaped_arg'"
done

if [ "$USE_OSG" = "1" ]; then
  WANT_GLIDEIN="+WantGlidein = true"
fi

#       Ensure that your environment is correct

if ! [ -d "$CMSSW_HOME" ]; then
  die "No such directory: $CMSSW_HOME"
fi

# randomize
if [ "$RANDOM" = "" ]; then
  die "\$RANDOM is not working.  Are you using bash?"
fi
RANDOM=`date +%N | cut -c 1-6 | sed 's|^0*||'`

proxy=${X509_USER_PROXY:-/tmp/x509up_u$UID}

if [ "$NO_SUBMIT" != 1 ] && ! check_proxy $MIN_PROXY_HOURS $proxy; then
  logerror
  logerror "Either rerun this command with --no-submit or create a new grid proxy"
  logerror "and rerun this command.  Example of how to create a grid proxy:"
  logerror
  logerror "voms-proxy-init --voms=cms --valid=48:00"
  die
fi

for exe in "${FARMOUT_HOOK_PRERUN}" "${FARMOUT_HOOK_POSTRUN}"; do
    [ -z "$exe" ] && continue
    if [ ! -x "${exe}" ]; then
        logerror "ERROR: Hook is not executable: $exe"
        die
    fi
    EXTRA_INPUTS="$EXTRA_INPUTS,$exe"
done

if [ "$EXTRA_INPUTS" != "" ]; then
  for file in ${EXTRA_INPUTS//,/ }; do
    if ! [ -a $file ]; then
      logerror "ERROR: Cannot find file specified in --extra-inputs: $file"
      die
    fi
  done
fi
export FARMOUT_HOOK_PRERUN
export FARMOUT_HOOK_POSTRUN

# Check the config template

if [ "$configTemplate" = "" ]; then
  die "You must specify a cfg template."
fi

CFG_EXTENSION="${configTemplate/*./}"
if [ "$CFG_EXTENSION" != "py" ] && [ "$CFG_EXTENSION" != "cfg" ]; then
  die "cfg template must end in .py or .cfg"
fi

for macro in \$randomNumber \$nEventsPerJob \$outputFileName; do
  if ! grep -F -q $macro $configTemplate && ! ( echo "$EXTRA_CMSRUN_ARGS" | grep -F -q $macro ); then
    die "$macro must appear in the configuration template or in the extra cmsRun arguments.  I can't find it in $configTemplate or the cmsRun arguments, so I am going to abort.  If you know what you are doing and this macro really isn't needed, simply put this macro in a comment in your template so that I see it and conclude that all is well."
  fi
done

# Check the chained config templates, if any
for cfg in ${chainedConfigs}; do
  ext="${cfg/*./}"
  if [ "$ext" != "$CFG_EXTENSION" ]; then
    die "The chained config template $cfg is expected to be the same type of config file as the base config file (${CFG_EXTENSION})."
  fi
  for macro in \$inputFileNames \$outputFileName; do
    # Currently, macros in arguments to chained runs are not supported,
    # so require the macros to appear in the chained config files.
    if ! grep -F -q $macro $cfg; then
      die "$macro must appear in the chained configuration template.  I can't find it in $cfg"
    fi
  done
done

# Note: reverse sort order is _very_ important, or the search/replace
# operation will do the wrong thing on longer macros having a common
# prefix with a shorter macro.
randomMacros=`grep -o '\$randomNumber[0-9]*' $configTemplate | sort -r | uniq`
randomMacrosInArgs=`echo "$EXTRA_CMSRUN_ARGS" | grep -o '\$randomNumber[0-9]*'  | sort -r | uniq`

#
# Environment setup
#
originalDir=`pwd`
PATH=$PATH:$originalDir
export PATH
cd $CMSSW_HOME || die "Failed to cd to $CMSSW_HOME."
eval `scramv1 runtime -sh`

if [ "$?" != "0" ]; then
  die "Failed to initialize CMSSW environment with scram in $CMSSW_HOME."
fi

for mr_name in ${!SITE_ENV_MATCH_REPLACE*}; do
    mr="${!mr_name}"
    # apply search-replace to the environment
    eval `env | sed "$mr" | sed 's/"/\\\\"/g' | sed 's/\\([^=]*\\)=\\(.*\\)/export \\1="\\2"/'`
done

OUTPUT_DIR=${OUTPUT_DIR:-${SRM_HOME}/$jobName}
SUBMIT_DIR=${SUBMIT_DIR:-${SUBMIT_HOME}/$jobName}
submitFile=$SUBMIT_DIR/submit
userCodeTgz=$SUBMIT_DIR/user_code.tgz
farmoutLogFile=$SUBMIT_DIR/farmoutRandomSeedJobs.log

if [ -d "$SUBMIT_DIR" ] && [ "$SKIP_EXISTING_JOBS" != "1" ]; then
  logerror
  logerror "Error: Submit directory already exists: $SUBMIT_DIR"
  logerror
  logerror "You must either remove it, or specify --skip-existing-jobs, or"
  logerror "specify a different job name or submission directory with --submit-dir"
  die
fi

mkdir -p $SUBMIT_DIR
cd $SUBMIT_DIR || die "Failed to create directory $SUBMIT_DIR"

SHARED_LOGS=$SHARED_LOGS/$(basename $SUBMIT_DIR)
mkdir -p $SHARED_LOGS

#
# Job specification
#
Executable=`which cmsRun.sh`

if [ "$QUICK_TEST" != "" ]; then
  IS_FAST_QUEUE_JOB="+IsFastQueueJob = True"
  SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && IsFastQueueSlot =?= True"
fi

#
# CMS Dashboard parameters
#
CMS_DASHBOARD_REPORTER_TGZ="${FARMOUT_HOME}/cmsdashboard_reporter.tgz"
FARMOUT_DASHBOARD_REPORTER="${FARMOUT_HOME}/farmout_dashboard.sh"
if ! [ -x "$FARMOUT_DASHBOARD_REPORTER" ] || ! [ -a "$CMS_DASHBOARD_REPORTER_TGZ" ]; then
   echo "No farmout_dashboard.sh or cmsdashboard_reporter.tgz found, so no reporting to the CMS dashboard."
   FARMOUT_DASHBOARD_REPORTER=""
   CMS_DASHBOARD_REPORTER_TGZ=""
fi
dboard="
dboard_taskId=${FARMOUT_USER}-`hostname -f`-\$(Cluster)
dboard_jobId=\$(Process)
dboard_sid=${FARMOUT_USER}-`hostname -f`-\$(Cluster).\$(Process)
dboard_application=`basename ${CMSSW_HOME}`
dboard_exe=cmsRun
dboard_tool=farmout
dboard_scheduler=local-condor
dboard_taskType=simulation
dboard_broker=local-condor-`hostname -f`
dboard_user=${FARMOUT_USER}
dboard_SyncCE=${CMS_DASHBOARD_LOCAL_CE}
"
# convert newlines to spaces
dboard="`echo $dboard`"

if [ "$SAVE_FAILED_DATAFILES" != "" ]; then
  #cmsRun.sh checks for this in the environment
  save_failed_datafiles_env="SAVE_FAILED_DATAFILES=1"
fi

if [ "$REQUIRES_WHOLE_MACHINE" == "1" ]; then
  REQUIRES_WHOLE_MACHINE_ATTR="+RequiresWholeMachine=true"

  #Note: the reference to per-slot Memory and Disk below are to prevent
  #condor from inserting the default requirements for these.  Instead,
  #we want to look at whole machine attributes.
  SITE_REQUIREMENTS="${SITE_REQUIREMENTS} && TARGET.CAN_RUN_WHOLE_MACHINE && TARGET.TotalMemory*1024 >= MY.ImageSize && TARGET.Memory > 0 && TARGET.TotalDisk >= MY.DiskUsage && TARGET.Disk > 0"
fi

if [ "$VSIZE_LIMIT" != "" ]; then
  vsize_env="FARMOUT_VSIZE_LIMIT=${VSIZE_LIMIT}"
  if [ "$VSIZE_LIMIT" -lt "$MEMORY_REQUIREMENTS" ]; then
    echo
    echo "WARNING: --vsize-limit=$VSIZE_LIMIT is smaller than --memory-requirements=$MEMORY_REQUIREMENTS"
    echo
  fi
fi

if [ "$NO_SHARED_FS" != 1 ]; then
  do_getenv="true"
  requires_shared_fs="true"
  dboard="${dboard} CMS_DASHBOARD_REPORTER_TGZ=${CMS_DASHBOARD_REPORTER_TGZ}"
  dboard="${dboard} FARMOUT_DASHBOARD_REPORTER=${FARMOUT_DASHBOARD_REPORTER}"
  checkSharedFS $CMSSW_HOME
else
  do_getenv="false"
  requires_shared_fs="false"

  cmssw_env="DO_RUNTIME_CMSSW_SETUP=1"

  if [ "$VO_CMS_SW_DIR" = "" ]; then
    die "VO_CMS_SW_DIR is not defined"
  fi
  cmssw_env="$cmssw_env VO_CMS_SW_DIR=${VO_CMS_SW_DIR}"

  if [ "$CMSSW_VERSION" = "" ]; then
    die "CMSSW_VERSION is not defined"
  fi
  cmssw_env="$cmssw_env CMSSW_VERSION=${CMSSW_VERSION}"

  cmssw_env="$cmssw_env SCRAM_ARCH=$(scramv1 arch)"

  if [ "$EXTRA_INPUTS" != "" ]; then
    EXTRA_INPUTS="${EXTRA_INPUTS},"
  fi

  cmssw_env="$cmssw_env CMSSW_USER_CODE_TGZ=$(basename $userCodeTgz)"
  packUserCode "${CMSSW_HOME}" "$userCodeTgz"
  EXTRA_INPUTS="${EXTRA_INPUTS}${userCodeTgz}"

  if ! [ -z "${FARMOUT_DASHBOARD_REPORTER}" ]; then
    dboard="${dboard} CMS_DASHBOARD_REPORTER_TGZ=$(basename ${CMS_DASHBOARD_REPORTER_TGZ})"
    dboard="${dboard} FARMOUT_DASHBOARD_REPORTER=$(basename ${FARMOUT_DASHBOARD_REPORTER})"
    EXTRA_INPUTS="${EXTRA_INPUTS},${FARMOUT_DASHBOARD_REPORTER},${CMS_DASHBOARD_REPORTER_TGZ}"
  fi

fi

# First put all the submit file commands that are the same for all jobs.
    cat <<EOF > $submitFile
X509UserProxy        = ${proxy}
Universe             = vanilla
Executable           = $Executable
GetEnv               = ${do_getenv}
# tell glideins to run job with access to cvmfs (via parrot)
+RequiresCVMFS       = True
# for reference by our own requirements expression
+RequiresSharedFS    = ${requires_shared_fs}
${WANT_GLIDEIN}
Environment          = "${dboard} ${save_failed_datafiles_env} ${vsize_env} ${cmssw_env}"
Copy_To_Spool        = false
Notification         = never
WhenToTransferOutput = On_Exit
ShouldTransferFiles  = yes
on_exit_remove       = (ExitBySignal == FALSE && (ExitCode == 0 || ExitCode == ${FAIL_JOB} || NumJobStarts>3))
request_memory       = ${MEMORY_REQUIREMENTS}
request_disk         = $(($DISK_REQUIREMENTS*1024))
${IS_FAST_QUEUE_JOB}
Requirements         = ${SITE_REQUIREMENTS}
# stop jobs from running if they blow up
periodic_hold        = DiskUsage/1024 > ${HOLD_DISK_USAGE_FACTOR}*${DISK_REQUIREMENTS}
job_ad_information_attrs = MachineAttrGLIDEIN_Site0,MachineAttrName0
${ACCOUNTING_GROUP}
${REQUIRES_WHOLE_MACHINE_ATTR}
EOF

# The following periodic hold expression caused jobs to get put on hold
# when condor happened to monitor the image size when the srm client
# was running, because the new version of the 64-bit jvm has a default
# 4GB heap size.
#ImageSize/1024 > ${HOLD_IMAGE_SIZE_FACTOR}*${MEMORY_REQUIREMENTS}

#
# Starting values for the job loop
#
declare -i nEventsSubmitted=0
declare -i job=0

final_step=1
for template in $chainedConfigs; do
  let final_step=$final_step+1
done

output_file_count=0

#
# Loop over jobs
#
while (( $nEvents > $nEventsSubmitted )); do
#
# Name the files
#
    job_output_dir="$OUTPUT_DIR"
    if [ "$OUTPUT_DIR" != "." ]; then
        if [ "$OUTPUT_FILES_PER_SUBDIR" != "0" ]; then
            job_output_dir="$job_output_dir/"$(($output_file_count/$OUTPUT_FILES_PER_SUBDIR+1))
        fi
    fi
    output_file_count=$(($output_file_count+1))

    jobtag=$jobName-`printf "%4.4d" $job`
    let job=$job+1
    if [ "$SKIP_EXISTING_JOBS" = "1" ] && [ -d $jobtag ]; then
      continue
    fi

    conlog=$jobtag.log
    stdout=$jobtag.out
    stderr=$jobtag.err
    jobcfg=$jobtag.$CFG_EXTENSION
    outputFileName=$jobtag.root

    if [ "$SKIP_EXISTING_OUTPUT" = "1" ]; then
      # Check for existing output file
      if outputFileExists $job_output_dir/$outputFileName; then
        continue
      fi
    fi

    step1_outputFileName=$outputFileName
    if [ "$chainedConfigs" != "" ]; then
      jobcfg=$jobtag-step1.$CFG_EXTENSION
      step1_outputFileName=intermediate/$jobtag-step1.root
    fi

    randomSed=""
    # Note: we rely here upon the reverse sort order of the randomMacros
    # in order to assure proper treatment of macros that share a common
    # prefix (e.g. randomNumber11 should be replaced before randomNumber1)
    for randomMacro in $randomMacros; do
        # ${RANDOM} is 15-bit
        # prepend the job number to it to make sure no two jobs in
        # a batch have the same seed
	# The modulo 900000000 is to avoid going over the limit
	# for maximum seed value in HepJamesRandom.
        randomNumber=$(( (${RANDOM} + (${job}<<15)) % 900000000 ))

        randomSed="${randomSed}s/\\${randomMacro}/${randomNumber}/g;"
    done
    randomSedInArgs=""
    for randomMacro in $randomMacrosInArgs; do
        randomNumber=$(( (${RANDOM} + (${job}<<15)) % 900000000 ))

        randomSedInArgs="${randomSedInArgs}s/\\${randomMacro}/${randomNumber}/g;"
    done

#
# Create the job subdirectory
#
    mkdir -p $jobtag || die "Failed to mkdir $jobtag"

#
# Prepare job configuration file
#

    sed < $configTemplate \
      "${randomSed}
       s|\\\$nEventsPerJob|$nEventsPerJob|g;
       s|\\\$outputFileName|$step1_outputFileName|g;
       s|\\\$jobNumber|$(($job-1))|g" > $jobtag/$jobcfg

    job_extra_args=""
    if [ "$EXTRA_CMSRUN_ARGS" != "" ]; then
      escaped_outputFileName=$(echo "$outputFileName" | sed "s|'|''|g; s|\"|\"\"|g")
      job_extra_args=$(echo "$EXTRA_CMSRUN_ARGS" | sed \
      "${randomSedInArgs}
       s|\\\$nEventsPerJob|$nEventsPerJob|g;
       s|\\\$outputFileName|$step1_outputFileName|g;
       s|\\\$jobNumber|$(($job-1))|g" )
    fi

    jobinputfiles="$jobcfg"
    jobcfgs="$jobcfg"

    last_outputFileName=$step1_outputFileName
    step=2
    for template in $chainedConfigs; do
      step_cfg=$jobtag-step${step}.$CFG_EXTENSION
      jobcfgs="$jobcfgs,$step_cfg"
      step_outputFileName=intermediate/$jobtag-step${step}.root
      if [ $step -eq $final_step ]; then
        step_outputFileName=$outputFileName
      fi
      jobinputfiles="$jobinputfiles,$step_cfg"

      step_inputFileNames="\"file:${last_outputFileName}\""
      sed < $template \
          "s|\\\$inputFileNames|$step_inputFileNames|g;
           s|\\\$outputFileName|$step_outputFileName|g" > $jobtag/$step_cfg


      last_outputFileName=$step_outputFileName
      let step=$step+1
    done

    if [ "$EXTRA_INPUTS" != "" ]; then
      jobinputfiles="$jobinputfiles,$EXTRA_INPUTS"
    fi

#
# Prepare condor submit commands for the job
#
    cat <<EOF >> $submitFile

InitialDir           = $jobtag
Arguments            = "$jobcfgs $outputFileName $job_output_dir$job_extra_args"
Transfer_Input_Files = $jobinputfiles
output               = $stdout
error                = $stderr
Log                  = $conlog
Queue
EOF
#
# Prepare for the next job
#
    let nEventsSubmitted=$nEventsSubmitted+$nEventsPerJob

    touch $jobtag/$stdout $jobtag/$stderr $jobtag/$conlog
    ln -f $jobtag/$stdout $SHARED_LOGS/$stdout
    ln -f $jobtag/$stderr $SHARED_LOGS/$stderr
    ln -f $jobtag/$conlog $SHARED_LOGS/$conlog

done


#
# Submit the jobs
#
if [ -z "$NO_SUBMIT" ]; then
    # The job is messed up if X509_USER_PROXY is defined, because then
    # Condor doesn't override this value to point to the actual proxy
    # location on the execution node.
    unset X509_USER_PROXY

    condor_submit $submitFile >> $farmoutLogFile
    cat $farmoutLogFile
else
    echo "Submit file $submitFile has been created but not submitted."
fi

echo "Jobs for $nEventsSubmitted events of $jobName are created in $SUBMIT_DIR"

echo ""
echo "Your jobs should show up in ~6 minutes"
echo "at the NEW job monitoring web page :"
echo "    http://www.hep.wisc.edu/cms/comp/jobs/"