Feature/mesoscale restart NAM/RAP stats

parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.nam

@@ -15,6 +15,10 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})"
 export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output
 export MET_CONFIG_OVERRIDES=""
 
+# Restart Settings
+export RESTART_DIR="${COMOUTsmall}/restart/c${vhr}"
+export COMPLETED_JOBS_FILE="completed_jobs.txt"
+
 # Time Settings
 export FHR_START=0
 export FHR_INCR=6

parm/evs_config/mesoscale/config.evs.prod.stats.mesoscale.atmos.grid2obs.rap

@@ -15,6 +15,10 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})"
 export MET_PLUS_OUT=${DATA}/${VERIF_CASE}/METplus_output
 export MET_CONFIG_OVERRIDES=""
 
+# Restart Settings
+export RESTART_DIR="${COMOUTsmall}/restart/c${vhr}"
+export COMPLETED_JOBS_FILE="completed_jobs.txt"
+
 # Time Settings
 export FHR_START=0
 export FHR_INCR=1

scripts/stats/mesoscale/exevs_mesoscale_nam_grid2obs_stats.sh

@@ -42,6 +42,7 @@ echo "*****************************"
 # Reformat MET Data
   export job_type="reformat"
   export njob=1
+  export run_restart=true
   for NEST in $NEST_LIST; do
      export NEST=$NEST
      for VERIF_TYPE in $VERIF_TYPES; do
@@ -55,8 +56,12 @@ echo "*****************************"
       # Check for restart files reformat
         echo " Check for restart files reformat begin"
         if [ $evs_run_mode = production ]; then
-           ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh
-	   export err=$?; err_chk
+	 # Check For Restart Files
+         if [ "$run_restart" = true ]; then
+	      python ${USHevs}/mesoscale/mesoscale_production_restart.py
+	      export err=$?; err_chk
+	      export run_restart=false
+         fi
         fi
         echo " Check for restart files reformat done"
 

scripts/stats/mesoscale/exevs_mesoscale_rap_grid2obs_stats.sh

@@ -43,6 +43,7 @@ echo "*****************************"
 # Reformat MET Data
   export job_type="reformat"
   export njob=1
+  export run_restart=true
   for NEST in $NEST_LIST; do
      export NEST=$NEST
      for VERIF_TYPE in $VERIF_TYPES; do
@@ -58,8 +59,12 @@ echo "*****************************"
       # Check for restart files reformat
         echo " Check for restart files reformat begin"
         if [ $evs_run_mode = production ]; then
-	   ${USHevs}/mesoscale/mesoscale_stats_g2o_production_restart.sh
-	   export err=$?; err_chk
+         # Check For Restart Files
+	 if [ "$run_restart" = true ]; then
+	     python ${USHevs}/mesoscale/mesoscale_production_restart.py
+	     export err=$?; err_chk
+	     export run_restart=false
+	 fi
         fi
         echo " Check for restart files reformat done"
 

ush/mesoscale/mesoscale_check_settings.py

@@ -58,7 +58,7 @@
         'MET_PLUS_CONF','MET_PLUS_OUT',
         'LOG_MET_OUTPUT_TO_METPLUS','NEST','TEMP_DIR','GRID_DIR','URL_HEAD',
         ]
-evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['bufr_ROOT']
+evs_mesoscale_settings_dict['RUN_GRID2OBS_STATS'] = ['RESTART_DIR', 'bufr_ROOT']
 evs_mesoscale_settings_dict['RUN_GRID2OBS_PLOTS'] = [
         'MET_VERSION','IMG_HEADER','PRUNE_DIR','SAVE_DIR','LOG_TEMPLATE',
         'LOG_LEVEL','STAT_OUTPUT_BASE_DIR','STAT_OUTPUT_BASE_TEMPLATE',

ush/mesoscale/mesoscale_create_output_dirs.py

@@ -85,6 +85,7 @@
         COMOUTplots = os.environ['COMOUTplots']
 if STEP == 'stats':
     job_type = os.environ['job_type']
+    RESTART_DIR = os.environ['RESTART_DIR']
 if STEP == 'plots':
     RESTART_DIR = os.environ['RESTART_DIR']
 
@@ -230,23 +231,39 @@
             working_output_base_dir = os.path.join(
                 DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE
             )
+            COMOUT_restart_base_dir = os.path.join(
+                RESTART_DIR, 'METplus_output', VERIF_TYPE
+            )
         if job_type == 'generate':
             working_output_base_dir = os.path.join(
                 DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE
             )
+            COMOUT_restart_base_dir = os.path.join(
+                RESTART_DIR, 'METplus_output', VERIF_TYPE
+            )
         if job_type == 'gather':
             working_output_base_dir = os.path.join(
                 DATA, VERIF_CASE, 'METplus_output', 'gather_small'
             )
+            COMOUT_restart_base_dir = os.path.join(
+                RESTART_DIR, 'METplus_output', 'gather_small'
+            )
         if job_type == 'gather2':
             working_output_base_dir = os.path.join(
                 DATA, VERIF_CASE, 'METplus_output'
             )
+            COMOUT_restart_base_dir = os.path.join(
+                RESTART_DIR, 'METplus_output'
+            )
         if job_type == 'gather3':
             working_output_base_dir = os.path.join(
                 DATA, VERIF_CASE, 'METplus_output'
             )
+            COMOUT_restart_base_dir = os.path.join(
+                RESTART_DIR, 'METplus_output'
+            )
         working_dir_list.append(working_output_base_dir)
+        COMOUT_dir_list.append(COMOUT_restart_base_dir)
         if job_type == 'reformat':
             working_dir_list.append(os.path.join(
                 working_output_base_dir, NEST, 'pb2nc', 'confs'
@@ -257,6 +274,9 @@
             working_dir_list.append(os.path.join(
                 working_output_base_dir, NEST, 'pb2nc', 'tmp'
             ))
+            COMOUT_dir_list.append(os.path.join(
+                COMOUT_restart_base_dir, NEST, 'pb2nc'
+            ))
             if NEST in ['spc_otlk', 'firewx']:
                 working_dir_list.append(os.path.join(
                     working_output_base_dir, 'genvxmask', 'confs'
@@ -271,6 +291,10 @@
                     working_output_base_dir, 'genvxmask',
                     NEST+'.'+vdate_dt.strftime('%Y%m%d')
                 ))
+                COMOUT_dir_list.append(os.path.join(
+                    COMOUT_restart_base_dir, 'genvxmask',
+                    NEST+'.'+vdate_dt.strftime('%Y%m%d')
+                ))
         if job_type == 'generate':
             working_dir_list.append(os.path.join(
                 working_output_base_dir, 'regrid_data_plane', 'confs'
@@ -285,6 +309,10 @@
                 working_output_base_dir, 'regrid_data_plane',
                 MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
                 ))
+            COMOUT_dir_list.append(os.path.join(
+                COMOUT_restart_base_dir, 'regrid_data_plane',
+                MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
+                ))
             working_dir_list.append(os.path.join(
                 working_output_base_dir, 'point_stat', 'confs'
             ))
@@ -298,6 +326,10 @@
                 working_output_base_dir, 'point_stat', 
                 MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
             ))
+            COMOUT_dir_list.append(os.path.join(
+                COMOUT_restart_base_dir, 'point_stat', 
+                MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
+            ))
         if job_type in ['gather', 'gather2', 'gather3']:
             working_dir_list.append(os.path.join(
                 working_output_base_dir, 'stat_analysis', 'confs'
@@ -312,6 +344,10 @@
                 working_output_base_dir, 'stat_analysis', 
                 MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
             ))
+            COMOUT_dir_list.append(os.path.join(
+                COMOUT_restart_base_dir, 'stat_analysis', 
+                MODELNAME+'.'+vdate_dt.strftime('%Y%m%d')
+            ))
         COMOUT_dir_list.append(os.path.join(
             COMOUTsmall,
             'gather_small'

ush/mesoscale/mesoscale_production_restart.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
-'''
-Name: mesoscale_production_restart.py
-Contact(s): Marcel Caron
-Abstract:
-'''
+# =============================================================================
+#
+# NAME: mesoscale_production_restart.py
+# CONTRIBUTOR(S): Marcel Caron, [email protected], NOAA/NWS/NCEP/EMC-VPPPGB
+# PURPOSE: Check the appropriate restart directory for restart files and copy
+#          the available files to the working directory
+#
+# =============================================================================
 
 import os
 import glob
@@ -20,20 +23,33 @@
 NET = os.environ['NET']
 RUN = os.environ['RUN']
 COMPONENT = os.environ['COMPONENT']
-VERIF_CASE = os.environ['VERIF_CASE']
 STEP = os.environ['STEP']
+VERIF_CASE = os.environ['VERIF_CASE']
 
 # Copy files for restart
-if STEP == 'plots':
+if STEP == 'stats':
+    VERIF_CASE = os.environ['VERIF_CASE']
+    RESTART_DIR = os.environ['RESTART_DIR']
+    working_dir = os.path.join(DATA, VERIF_CASE)
+    completed_jobs_file = os.path.join(RESTART_DIR, 'completed_jobs.txt')
+    if os.path.exists(RESTART_DIR):
+        if (os.path.exists(completed_jobs_file) 
+                and os.stat(completed_jobs_file).st_size != 0):
+            print(f"Copying restart directory {RESTART_DIR} "
+                  +f"into working directory {working_dir}")
+            cutil.run_shell_command(
+                ['cp', '-rpv', RESTART_DIR, working_dir]
+            )
+elif STEP == 'plots':
     COMOUTplots = os.environ['COMOUTplots']
     RESTART_DIR = os.environ['RESTART_DIR']
-    COMPLETED_JOBS_FILE = os.environ['COMPLETED_JOBS_FILE']
-    working_dir = os.path.join(DATA, VERIF_CASE, 'out')
-    completed_jobs_file = os.path.join(RESTART_DIR, COMPLETED_JOBS_FILE)
+    SAVE_DIR = os.environ['SAVE_DIR']
+    completed_jobs_file = os.path.join(RESTART_DIR, f'completed_jobs.txt')
     if os.path.exists(completed_jobs_file):
         if os.stat(completed_jobs_file).st_size != 0:
             cutil.run_shell_command(
-                ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), working_dir]
+                ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), SAVE_DIR]
             )
 
+
 print("END: "+os.path.basename(__file__))