heterogeneous-cluster-site-config

# This file contains all the site specific configurations,
# like for example the various paths, which partition to be used,
# the amount of requestable resources like cores, memory and time.
# Others may be added there as well.
# The idea is to use that file so all the configurations are the same,
# regardless whether they are in the Singularity Definition file or in
# the installation script. 

# Variables PBSpro concerning
# The 'PLATFORM' is the various platforms which are being build
# with the functions below the definition of them
PLATFORMS="icelake rome"
GPUPLATFORMS="icelake-gpu"

# We need to specify the queue:
QUEUE="cx"

# We define which softwarestack we are building. The default for prod is using
# the EasyBuild supplied flags. We can overwrite that for specific partitions of
# PBS further below using that syntax.
# EASYBUILD_OPTARCH=""
# We do NOT define a default!

function icelake {
PARTITION=":cpu_type=icelake"
CORES="24"
ARCH="icelake"
}

function rome {
PARTITION=":cpu_type=rome"
CORES="24"
ARCH="zen2"
}

function dev {
PARTITION=""
CORES="24"
ARCH="icelake"
EASYBUILD_OPTARCH="Intel:xAVX;GCC:march=nocona -mtune=generic"
SOFTWARE_HOME="/rds/easybuild/dev/apps/"
EASYBUILD_PREFIX="/rds/easybuild/dev/apps/"
EASYBUILD_INSTALLPATH="/rds/easybuild/dev/apps/"
MODULEPATH="/rds/easybuild/dev/apps/modules/all"
}

#### GPU SECTION ###
# We need to set the version of the nVidia GPUs we are using
# For the RTX6000 that is 7.5
# For the A100 that is 8.0
# See here for more details:
# https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/

function icelake-gpu {
PARTITION=":cpu_type=icelake:ngpus=2:gpu_type=A100"
CORES="12"
ARCH="icelake"
CUDA_COMPUTE_CAPABILITIES="8.0"
}

function rome-gpu {
PARTITION=":cpu_type=rome:ngpus=4:gpu_type=RTX6000"
CORES="24"
ARCH="zen2"
CUDA_COMPUTE_CAPABILITIES="7.5"
}

###

# What is it we want to use to install? Only EasyBuild or EESSI or both?
# Possible choices are EB, EESSI and BOTH 
# Right now, only the EB section is actually working
INSTALLING="EB"

# These bits are needed for the software installation using EasyBuild:

# Where to install the software. These are the paths OUTSIDE the container!
# Change as required
SOFTWARE_INSTDIR="/rds/easybuild"
SOFTWARE_HOME="${SOFTWARE_INSTDIR}/${ARCH}/apps"
# Which container name to be used:
CONTAINER_DIR="${SOFTWARE_INSTDIR}/containers"
CONTAINER_VERSION="eb-4.8.0-lmod-rocky8.7-cx.sif"
# For testing we are using a different container: 
CONTAINER_TESTING_VERSION="eb-4.7.0-lmod-alma8.8.sif"
# We might need to bind an additional external directory into the container:
BINDDIR="${SOFTWARE_INSTDIR}:/software"
# We need that so we got the PBS-Pro stuff inside the container
OPTDIR="/opt:/opt"
# The CUDA-installer wants to write to /var/log, so we do this
CUDALOGS="${TMPDIR}/cuda-logs"


# Variables EasyBuild concerning, i.e. used INSIDE the container:
# Some variables need to come from PBSpro above!
EASYBUILD_ACCEPT_EULA_FOR="Intel-oneAPI,NVHPC,NVIDIA-driver"
EASYBUILD_PREFIX="/sw-eb"
EASYBUILD_SOURCEPATH="/software/sources"
EASYBUILD_INSTALLPATH="/sw-eb"
EASYBUILD_BUILDPATH="/dev/shm/$USER"
EASYBUILD_TMPDIR="/dev/shm/$USER"
EASYBUILD_PARALLEL="$CORES"
EB_VERSION="4.9.1"
MODULEPATH="/sw-eb/modules/all"
EB="eb --trace --robot --download-timeout=100 --hooks=/software/hooks/site-hooks.py"
# No nooks if we are testing!
# EB="eb --trace --robot --download-timeout=100"