diff --git a/env/ORION.env b/env/ORION.env index 400963d3c8..e20945bc07 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -132,20 +132,20 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export NTHREADS_GSI=${nth_anal:-${nth_max}} [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max} - export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}" + export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}" export NTHREADS_CALCINC=${nth_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max} - export APRUN_CALCINC="${launcher} \$ncmd" + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" export NTHREADS_CYCLE=${nth_cycle:-12} [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max} npe_cycle=${ntiles:-6} - export APRUN_CYCLE="${launcher} -n ${npe_cycle}" + export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}" export NTHREADS_GAUSFCANL=1 npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}" + export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" elif [[ "${step}" = "sfcanl" ]]; then nth_max=$((npe_node_max / npe_node_sfcanl)) diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh index c91b9d4679..ca8bd7aa24 100755 --- a/jobs/rocoto/post.sh +++ b/jobs/rocoto/post.sh @@ -8,9 +8,26 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -status=$? -[[ ${status} -ne 0 ]] && exit ${status} +# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh +# status=$? +# [[ ${status} -ne 0 ]] && exit ${status} +# Temporarily load modules from UPP +source "${HOMEgfs}/ush/detect_machine.sh" +source "${HOMEgfs}/ush/module-setup.sh" +module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles" +module load "${MACHINE_ID}" +module load prod_util +if [[ "${MACHINE_ID}" = "wcoss2" ]]; then + module load cray-pals + module load cfp +else + # shellcheck disable=SC2154 + export UTILROOT="${prod_util_ROOT}" +fi +module load grib-util +module load wgrib2 +export WGRIB2=wgrib2 +# End hack export job="post" export jobid="${job}.$$" diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 9dcf1f873b..884fe1723c 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -180,7 +180,7 @@ case "${CASE}" in export OCNRES=025 export waveGRD='glo_025' ;; - "C768") + "C768" | "C1152") export OCNRES=025 export waveGRD='mx025' ;; diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 6d081b3fe0..7f3956af4d 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -18,7 +18,7 @@ case "${CASE}" in export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored export CPL_ICEIC=CPC_refactored export CPL_OCNIC=CPC3Dvar_refactored - export CPL_WAVIC=GEFSwave20210528v2_refactored + export CPL_WAVIC=workflow_C384_refactored ;; "C768") export CPL_ATMIC=HR2_refactored @@ -26,6 +26,12 @@ case "${CASE}" in export CPL_OCNIC=HR1_refactored export CPL_WAVIC=HR1_refactored ;; + "C1152") + export CPL_ATMIC=HR2_C1152_refactored + export CPL_ICEIC=HR3_refactored + export CPL_OCNIC=HR3_refactored + export CPL_WAVIC=HR1_refactored + ;; *) echo "FATAL ERROR Unrecognized resolution: ${CASE}" exit 1 diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 3d2e36771a..2441716a05 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -185,7 +185,7 @@ case "${fv3_res}" in export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 - export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10 # TODO: refine these numbers when a case is available + export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 # TODO: refine these numbers when a case is available ;; "C3072") export DELTIM=90 diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 463d16a0fa..83b0866eb3 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -177,6 +177,13 @@ for checkout_pid in $(jobs -p); do wait "${checkout_pid}" || errs=$((errs + $?)) done +# Temporary hack to check out a UPP verison that works on Orion +# This can be removed once the UFS UPP version advances to or beyond 78f369b +cd "${topdir}/ufs_model.fd/FV3/upp" || exit 1 +git checkout 78f369b +cd "${topdir}" || exit 1 +# End hack + if (( errs > 0 )); then echo "WARNING: One or more errors encountered during checkout process, please check logs before building" fi diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 4279a99dd4..3f2dd9dcf9 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -117,3 +117,11 @@ elif [[ ${MACHINE_ID} = "noaacloud" ]]; then else echo WARNING: UNKNOWN PLATFORM 1>&2 fi + +# If this function exists in the environment, run it; else do not +ftype=$(type -t set_strict) +if [[ "${ftype}" == "function" ]]; then + set_strict +else + set +u +fi diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index cdb5e18f3e..3b8472d3c8 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -47,7 +47,7 @@ def _get_app_configs(self): if self.do_ocean: configs += ['ocnpost'] - configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'fit2obs', 'arch', 'cleanup'] + configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch', 'cleanup'] if self.do_hybvar: if self.do_jediatmens: @@ -56,6 +56,9 @@ def _get_app_configs(self): configs += ['eobs', 'eomg', 'ediag', 'eupd'] configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc'] + if self.do_fit2obs: + configs += ['fit2obs'] + if self.do_verfozn: configs += ['verfozn'] diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 31911f2d21..e44c0aece0 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -22,4 +22,4 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index 439b5e8825..459aee7cf6 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -22,4 +22,4 @@ LOCALARCH: 'NO' ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index 41e1044eff..04a5949b2e 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -22,4 +22,4 @@ LOCALARCH: 'NO' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 5e2ed8cd03..24d31521c2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -110,15 +110,17 @@ def waveinit(self): resources = self.get_resource('waveinit') dependencies = None + cycledef = None if self.app_config.mode in ['cycled']: deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) + if self.cdump in ['gdas']: + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - - task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump + task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) return task @@ -128,9 +130,9 @@ def waveprep(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('waveprep') - task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, cycledef=cycledef) return task @@ -488,6 +490,8 @@ def _fcst_forecast_only(self): wave_job = 'waveprep' if self.app_config.model_app in ['ATMW'] else 'waveinit' dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}waveinit'} + dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_aero: # Calculate offset based on CDUMP = gfs | gdas @@ -521,10 +525,6 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalpost'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: - dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} - dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero: dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlfinal'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -540,6 +540,11 @@ def _fcst_cycled(self): dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: + dep_dict = {'type': 'task', 'name': f'{self.cdump}waveprep'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('fcst') @@ -917,11 +922,8 @@ def fit2obs(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump - resources = self.get_resource('fit2obs') - task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, - cycledef=cycledef) + task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) return task @@ -952,33 +954,32 @@ def metp(self): def arch(self): deps = [] dependencies = [] - if self.app_config.do_verfozn or self.app_config.do_verfrad or self.app_config.do_vminmon: - if self.app_config.mode in ['cycled']: - if self.cdump in ['gfs']: - if self.app_config.do_vminmon: - dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'} - deps.append(rocoto.add_dependency(dep_dict)) - elif self.cdump in ['gdas']: - deps2 = [] - if self.app_config.do_verfozn: - dep_dict = {'type': 'task', 'name': f'{self.cdump}verfozn'} - deps2.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_verfrad: - dep_dict = {'type': 'task', 'name': f'{self.cdump}verfrad'} - deps2.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_vminmon: - dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'} - deps2.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps2) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + if self.app_config.mode in ['cycled']: + if self.cdump in ['gfs']: + if self.app_config.do_vminmon: + dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'} + deps.append(rocoto.add_dependency(dep_dict)) + elif self.cdump in ['gdas']: # Block for handling half cycle dependencies + deps2 = [] + if self.app_config.do_fit2obs: + dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'} + deps2.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_verfozn: + dep_dict = {'type': 'task', 'name': f'{self.cdump}verfozn'} + deps2.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_verfrad: + dep_dict = {'type': 'task', 'name': f'{self.cdump}verfrad'} + deps2.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_vminmon: + dep_dict = {'type': 'task', 'name': f'{self.cdump}vminmon'} + deps2.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps2) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) if self.app_config.do_vrfy: dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'} deps.append(rocoto.add_dependency(dep_dict)) - if self.cdump in ['gdas'] and self.app_config.do_fit2obs: - dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'} - deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave: dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostsbs'} deps.append(rocoto.add_dependency(dep_dict))