first draft

parm/soca/obsprep/obsprep_config.yaml

@@ -244,6 +244,9 @@ observations:
     provider: GTS
     dmpdir subdir: atmos
     type: bufr
+    window:
+       back: 4
+       forward: 4
     dmpdir regex: 'gdas.*.subpfl.*.bufr_d'
 
 - obs space:

ush/soca/prep_ocean_obs.py

@@ -147,7 +147,8 @@ def initialize(self):
                             logger.warning(f"No files found for obs source {obtype}, skipping")
                             break  # go to next observer in OBS_YAML
 
-                        obsprep_space['input files'] = input_files
+                        obsprep_space['input files'] = [f[0] for f in input_files]
+                        obsprep_space['input cycles'] = [f[1] for f in input_files]
                         obsprep_space['window begin'] = self.window_begin
                         obsprep_space['window end'] = self.window_end
                         ioda_filename = f"{RUN}.t{cyc:02d}z.{obs_space_name}.{cdatestr}.nc4"
@@ -164,13 +165,14 @@ def initialize(self):
                                 'COM_OBS': COMIN_OBS,
                                 'OCEAN_BASIN_FILE': OCEAN_BASIN_FILE}
                             obsprep_space['conversion config file'] = ioda_config_file
-                            bufr2iodapy = BUFR2IODA_PY_DIR + '/bufr2ioda_' + obtype + '.py'
+                            bufr2iodapy = os.path.join(BUFR2IODA_PY_DIR, f'bufr2ioda_{obtype}.py')
                             obsprep_space['bufr2ioda converter'] = bufr2iodapy
-                            tmpl_filename = 'bufr2ioda_' + obtype + '.yaml'
+                            tmpl_filename = f"bufr2ioda_{obtype}.yaml"
                             bufrconv_template = os.path.join(BUFR2IODA_TMPL_DIR, tmpl_filename)
 
                             try:
                                 bufrconv = parse_j2yaml(bufrconv_template, bufrconv_config)
+                                bufrconv.update(obsprep_space)
                                 bufrconv.save(ioda_config_file)
                             except Exception as e:
                                 logger.warning(f"An exeception {e} occured while trying to create BUFR2IODA config")

ush/soca/prep_ocean_obs_utils.py

@@ -2,7 +2,7 @@
 import os
 import fnmatch
 import subprocess
-from wxflow import FileHandler, Logger
+from wxflow import FileHandler, Logger, YAMLFile
 
 logger = Logger()
 
@@ -36,10 +36,10 @@ def obs_fetch(config, task_config, obsprep_space, cycles):
 
         for root, _, files in os.walk(full_input_dir):
             for filename in fnmatch.filter(files, dumpdir_regex):
-                target_file = PDY + cyc + '-' + filename
-                matching_files.append((full_input_dir, filename, target_file))
+                target_file = f"{PDY}{cyc}-{filename}"
+                matching_files.append((full_input_dir, filename, target_file, f"{PDY}{cyc}"))
 
-    for full_input_dir, filename, target_file in matching_files:
+    for full_input_dir, filename, target_file, _ in matching_files:
         file_path = os.path.join(full_input_dir, filename)
         file_destination = os.path.join(COMIN_OBS, target_file)
         file_copy.append([file_path, file_destination])
@@ -50,7 +50,7 @@ def obs_fetch(config, task_config, obsprep_space, cycles):
     FileHandler({'copy': file_copy}).sync()
 
     # return the modified file names for the IODA converters
-    return [f[2] for f in matching_files]
+    return [(f[2],f[3]) for f in matching_files]
 
 
 def run_netcdf_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC):
@@ -69,11 +69,25 @@ def run_netcdf_to_ioda(obsspace_to_convert, OCNOBS2IODAEXEC):
 def run_bufr_to_ioda(obsspace_to_convert):
     logger.info(f"running run_bufr_to_ioda on {obsspace_to_convert['name']}")
     bufrconv_yaml = obsspace_to_convert['conversion config file']
+    bufrconv_config = YAMLFile(bufrconv_yaml)
     bufr2iodapy = obsspace_to_convert['bufr2ioda converter']
-    try:
-        subprocess.run(['python', bufr2iodapy, '-c', bufrconv_yaml], check=True)
-        return 0
-    except subprocess.CalledProcessError as e:
-        logger.warning(f"bufr2ioda converter failed with error  >{e}<, \
-            return code {e.returncode}")
-        return e.returncode
+    RUN = bufrconv_config['cycle_type']
+    obtype = obsspace_to_convert['name']
+    for input_file in obsspace_to_convert['input files']:
+        input_file_cdate = input_file[0:8]
+        input_file_cyc = input_file[8:10]
+        input_file_datecycle = f"{input_file_cdate}{input_file_cyc}"
+        bufrconv_config['input_file'] = input_file
+        bufrconv_config['output_file'] = f"{RUN}.t{input_file_cyc}z.{obtype}.{input_file_datecycle}.nc4"
+        bufrconv_config['cycle_datetime'] = input_file_datecycle
+        config_filename = f"{input_file_datecycle}.{bufrconv_yaml}"
+        print(f"config_filename: {config_filename}")
+        bufrconv_config.save(config_filename)
+        try:
+    #        subprocess.run(['python', bufr2iodapy, '-c', bufrconv_yaml], check=True)
+            subprocess.run(['python', bufr2iodapy, '-c', config_filename], check=True)
+#            return 0
+        except subprocess.CalledProcessError as e:
+            logger.warning(f"bufr2ioda converter failed with error  >{e}<, \
+                return code {e.returncode}")
+ #           return e.returncode