From 1c8096f8799a58bfbbacbe0450d31bc27cb6b096 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Nov 2024 11:44:57 -0500 Subject: [PATCH 01/12] Add state of ground to increase the snow depth reports. --- .../snow/obs/config/bufr2ioda_mapping.yaml.j2 | 1 + parm/snow/obs/config/bufr_sfcsno_mapping.yaml | 17 +++++----- .../obs/config/ioda_bufr_python_encoder.py | 31 +++++++++++++++++++ parm/snow/obs/config/sfcsno_snow.yaml.j2 | 6 ++-- sorc/bufr-query | 2 +- 5 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 parm/snow/obs/config/ioda_bufr_python_encoder.py diff --git a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 index 9340cf507..fd0f1d3f5 100644 --- a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 +++ b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 @@ -2,3 +2,4 @@ mkdir: - '{{ DATA }}/obs' copy: - ['{{PARMgfs}}/gdas/snow/obs/config/bufr_sfcsno_mapping.yaml', '{{ DATA }}/obs/'] +- ['{{PARMgfs}}/gdas/snow/obs/config/ioda_bufr_python_encoder.py', '{{ DATA }}'] diff --git a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml index 00ab1bc7d..2fe12f39c 100644 --- a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml +++ b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml @@ -17,7 +17,6 @@ bufr: query: "[*/CLON, */CLONH]" stationIdentification: query: "*/RPID" - stationElevation: query: "[*/SELV, */HSMSL]" @@ -26,11 +25,8 @@ bufr: query: "[*/SNWSQ1/TOSD, */MTRMSC/TOSD, */STGDSNDM/TOSD]" transforms: - scale: 1000.0 - filters: - - bounding: - variable: totalSnowDepth - lowerBound: 0 - upperBound: 10000000 + groundState: + query: "[*/GRDSQ1/SOGR, */STGDSNDM/SOGR]" encoder: variables: @@ -65,7 +61,7 @@ encoder: coordinates: "longitude latitude" source: variables/stationIdentification longName: "Identification of Observing Location" - units: "m" + units: "index" # ObsValue - name: "ObsValue/totalSnowDepth" @@ -73,3 +69,10 @@ encoder: source: variables/totalSnowDepth longName: "Total Snow Depth" units: "mm" + + - name: "ObsValue/groundState" + coordinates: "longitude latitude" + source: variables/groundState + longName: "STATE OF THE GROUND" + units: "index" + diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/obs/config/ioda_bufr_python_encoder.py new file mode 100644 index 000000000..98cccf288 --- /dev/null +++ b/parm/snow/obs/config/ioda_bufr_python_encoder.py @@ -0,0 +1,31 @@ +import numpy as np +import bufr +from pyioda.ioda.Engines.Bufr import Encoder + + +def mask_container(container, mask): + new_container = bufr.DataContainer() + for var_name in container.list(): + print(f" ... variable name: {var_name} ...") + var = container.get(var_name) + paths = container.get_paths(var_name) + new_container.add(var_name, var[mask], paths) + + return new_container + +def create_obs_group(input_path): + YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" + container = bufr.Parser(input_path, YAML_PATH).parse() + + sogr = container.get('variables/groundState') + snod = container.get('variables/totalSnowDepth') + snod[(sogr < 10.0) | (sogr == 11.0) | (sogr == 15.0)] = 0.0 + container.replace('variables/totalSnowDepth', snod) + + print(f" ... Remove filled/missing snow values ...") + masked_container = mask_container(container, (~snod.mask)) + + encoder = Encoder(YAML_PATH) + data = next(iter(encoder.encode(masked_container).values())) + + return data diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 960ea8f64..7fe48e45f 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -5,9 +5,11 @@ halo size: 250e3 obsdatain: engine: - type: bufr + type: script + script file: "{{ DATA }}/ioda_bufr_python_encoder.py" + args: + input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' - mapping file: '{{ DATA }}/obs/bufr_sfcsno_mapping.yaml' obsdataout: engine: type: H5File diff --git a/sorc/bufr-query b/sorc/bufr-query index 97367fcd5..8bc28860d 160000 --- a/sorc/bufr-query +++ b/sorc/bufr-query @@ -1 +1 @@ -Subproject commit 97367fcd59adf4863aba1a52189e20f9f66451af +Subproject commit 8bc28860d562f9a53ffa86e7c5c5ae05c2c4e09f From 365675ddbb7d56d0de04aaa5341e17f572c5628e Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 12 Nov 2024 19:19:04 -0500 Subject: [PATCH 02/12] Update the defined index with zero snow depth. --- parm/snow/obs/config/ioda_bufr_python_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/obs/config/ioda_bufr_python_encoder.py index 98cccf288..1132b416f 100644 --- a/parm/snow/obs/config/ioda_bufr_python_encoder.py +++ b/parm/snow/obs/config/ioda_bufr_python_encoder.py @@ -19,7 +19,7 @@ def create_obs_group(input_path): sogr = container.get('variables/groundState') snod = container.get('variables/totalSnowDepth') - snod[(sogr < 10.0) | (sogr == 11.0) | (sogr == 15.0)] = 0.0 + snod[(sogr <= 11.0) | (sogr == 15.0)] = 0.0 container.replace('variables/totalSnowDepth', snod) print(f" ... Remove filled/missing snow values ...") From 0c025221a8b456f6a65d149bccaa93c4e672e3b9 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 13 Nov 2024 12:45:55 -0500 Subject: [PATCH 03/12] Move python code to parm/gdas/snow/ directory to aviod extra copy. --- parm/snow/{obs/config => }/ioda_bufr_python_encoder.py | 0 parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 | 1 - parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- 3 files changed, 1 insertion(+), 2 deletions(-) rename parm/snow/{obs/config => }/ioda_bufr_python_encoder.py (100%) diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/ioda_bufr_python_encoder.py similarity index 100% rename from parm/snow/obs/config/ioda_bufr_python_encoder.py rename to parm/snow/ioda_bufr_python_encoder.py diff --git a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 index fd0f1d3f5..9340cf507 100644 --- a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 +++ b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 @@ -2,4 +2,3 @@ mkdir: - '{{ DATA }}/obs' copy: - ['{{PARMgfs}}/gdas/snow/obs/config/bufr_sfcsno_mapping.yaml', '{{ DATA }}/obs/'] -- ['{{PARMgfs}}/gdas/snow/obs/config/ioda_bufr_python_encoder.py', '{{ DATA }}'] diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 7fe48e45f..9f2d9cd24 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -6,7 +6,7 @@ obsdatain: engine: type: script - script file: "{{ DATA }}/ioda_bufr_python_encoder.py" + script file: "{{ PARMgfs }}/gdas/snow/ioda_bufr_python_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' From e57ac67561baee301284815a0edc233836bb64a6 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 13 Nov 2024 18:29:36 -0500 Subject: [PATCH 04/12] Modif to only replace the missing/filled snod values. --- parm/snow/ioda_bufr_python_encoder.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/parm/snow/ioda_bufr_python_encoder.py b/parm/snow/ioda_bufr_python_encoder.py index 1132b416f..f8f9c0f58 100644 --- a/parm/snow/ioda_bufr_python_encoder.py +++ b/parm/snow/ioda_bufr_python_encoder.py @@ -6,7 +6,6 @@ def mask_container(container, mask): new_container = bufr.DataContainer() for var_name in container.list(): - print(f" ... variable name: {var_name} ...") var = container.get(var_name) paths = container.get_paths(var_name) new_container.add(var_name, var[mask], paths) @@ -14,15 +13,28 @@ def mask_container(container, mask): return new_container def create_obs_group(input_path): + """Create the ioda snow observations + This method: + - reads state of ground (sogr) and snow depth (snod) + - applys sogr conditions to the missing snod values + - removes the filled/missing snow values and creates the masked container + - encoders the new container. + + Parameters + ---------- + input_path + The input bufr file + """ + YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" container = bufr.Parser(input_path, YAML_PATH).parse() sogr = container.get('variables/groundState') snod = container.get('variables/totalSnowDepth') - snod[(sogr <= 11.0) | (sogr == 15.0)] = 0.0 + snod[(sogr <= 11.0) & snod.mask] = 0.0 + snod[(sogr == 15.0) & snod.mask] = 0.0 container.replace('variables/totalSnowDepth', snod) - print(f" ... Remove filled/missing snow values ...") masked_container = mask_container(container, (~snod.mask)) encoder = Encoder(YAML_PATH) From 244693c68d47087bf530cc2d117ad2989e5193d2 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 14 Nov 2024 17:45:49 -0500 Subject: [PATCH 05/12] Address reviewer's comments. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- .../ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename parm/snow/ioda_bufr_python_encoder.py => ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py (92%) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 9f2d9cd24..d17d3e9b0 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -6,7 +6,7 @@ obsdatain: engine: type: script - script file: "{{ PARMgfs }}/gdas/snow/ioda_bufr_python_encoder.py" + script file: "{{ USHgfs }}/bufr2ioda_sfcsno_bufr_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' diff --git a/parm/snow/ioda_bufr_python_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py similarity index 92% rename from parm/snow/ioda_bufr_python_encoder.py rename to ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index f8f9c0f58..9fdc59538 100644 --- a/parm/snow/ioda_bufr_python_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -37,7 +37,6 @@ def create_obs_group(input_path): masked_container = mask_container(container, (~snod.mask)) - encoder = Encoder(YAML_PATH) - data = next(iter(encoder.encode(masked_container).values())) + data = next(iter(Encoder(YAML_PATH).encode(masked_container).values())) return data From 76a5e94be2810d1a4273fe3b04a46a2664641195 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 14 Nov 2024 17:54:21 -0500 Subject: [PATCH 06/12] Fix py norms error. --- ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 9fdc59538..16a648acd 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -12,13 +12,14 @@ def mask_container(container, mask): return new_container + def create_obs_group(input_path): """Create the ioda snow observations This method: - reads state of ground (sogr) and snow depth (snod) - applys sogr conditions to the missing snod values - removes the filled/missing snow values and creates the masked container - - encoders the new container. + - encoders the new container. Parameters ---------- From 2a72e4dc15670bc99c3dc9e2e79a114a30bdbb5d Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 15 Nov 2024 12:07:49 -0500 Subject: [PATCH 07/12] Add the mapping_path indicating the script backend requirements. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 1 + ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index d17d3e9b0..83cab6f6e 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -9,6 +9,7 @@ script file: "{{ USHgfs }}/bufr2ioda_sfcsno_bufr_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' + mapping_path: '{{ DATA }}/obs/bufr_sfcsno_mapping.yaml' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsdataout: engine: diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 16a648acd..140f508f1 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -13,7 +13,7 @@ def mask_container(container, mask): return new_container -def create_obs_group(input_path): +def create_obs_group(input_path, mapping_path): """Create the ioda snow observations This method: - reads state of ground (sogr) and snow depth (snod) @@ -27,7 +27,7 @@ def create_obs_group(input_path): The input bufr file """ - YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" + YAML_PATH = mapping_path container = bufr.Parser(input_path, YAML_PATH).parse() sogr = container.get('variables/groundState') From fbd1bb960b94bb5f0d6674a0a7860dbbc0816f51 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 15 Nov 2024 12:14:44 -0500 Subject: [PATCH 08/12] Add comments --- ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 140f508f1..6c75df0ae 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -23,8 +23,9 @@ def create_obs_group(input_path, mapping_path): Parameters ---------- - input_path - The input bufr file + input_path: The input bufr file + mapping_path: The input bufr2ioda mapping file + """ YAML_PATH = mapping_path From 91f160e614f8abef21529a9611f8aab6105f0ab3 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 22 Nov 2024 15:36:13 -0500 Subject: [PATCH 09/12] Update the missing value for elevation from bufr file. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 83cab6f6e..947d6340e 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -57,7 +57,7 @@ where: - variable: name: MetaData/stationElevation - minvalue: -999.0 + maxvalue: 999999999.9 - filter: Domain Check # land only where: - variable: From 3c1dbf75d0e6f6578b5302c60a4eae0e39126d1e Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 24 Dec 2024 18:51:35 -0500 Subject: [PATCH 10/12] Update the scripts and sorc/bufr-query. --- parm/snow/obs/config/bufr_sfcsno_mapping.yaml | 1 + sorc/bufr-query | 2 +- .../bufr2ioda_sfcsno_bufr_encoder.py | 44 ---- ush/ioda/bufr2ioda/bufr_sfcsno.py | 189 ++++++++++++++++++ 4 files changed, 191 insertions(+), 45 deletions(-) delete mode 100644 ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py create mode 100755 ush/ioda/bufr2ioda/bufr_sfcsno.py diff --git a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml index 2fe12f39c..31d042f27 100644 --- a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml +++ b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml @@ -19,6 +19,7 @@ bufr: query: "*/RPID" stationElevation: query: "[*/SELV, */HSMSL]" + type: float # ObsValue totalSnowDepth: diff --git a/sorc/bufr-query b/sorc/bufr-query index 8bc28860d..9e595b5a3 160000 --- a/sorc/bufr-query +++ b/sorc/bufr-query @@ -1 +1 @@ -Subproject commit 8bc28860d562f9a53ffa86e7c5c5ae05c2c4e09f +Subproject commit 9e595b5a3bb91791f7f5ada456298e2b966e74f6 diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py deleted file mode 100644 index 6c75df0ae..000000000 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ /dev/null @@ -1,44 +0,0 @@ -import numpy as np -import bufr -from pyioda.ioda.Engines.Bufr import Encoder - - -def mask_container(container, mask): - new_container = bufr.DataContainer() - for var_name in container.list(): - var = container.get(var_name) - paths = container.get_paths(var_name) - new_container.add(var_name, var[mask], paths) - - return new_container - - -def create_obs_group(input_path, mapping_path): - """Create the ioda snow observations - This method: - - reads state of ground (sogr) and snow depth (snod) - - applys sogr conditions to the missing snod values - - removes the filled/missing snow values and creates the masked container - - encoders the new container. - - Parameters - ---------- - input_path: The input bufr file - mapping_path: The input bufr2ioda mapping file - - """ - - YAML_PATH = mapping_path - container = bufr.Parser(input_path, YAML_PATH).parse() - - sogr = container.get('variables/groundState') - snod = container.get('variables/totalSnowDepth') - snod[(sogr <= 11.0) & snod.mask] = 0.0 - snod[(sogr == 15.0) & snod.mask] = 0.0 - container.replace('variables/totalSnowDepth', snod) - - masked_container = mask_container(container, (~snod.mask)) - - data = next(iter(Encoder(YAML_PATH).encode(masked_container).values())) - - return data diff --git a/ush/ioda/bufr2ioda/bufr_sfcsno.py b/ush/ioda/bufr2ioda/bufr_sfcsno.py new file mode 100755 index 000000000..03a9102b4 --- /dev/null +++ b/ush/ioda/bufr2ioda/bufr_sfcsno.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +import sys +import os +import argparse +import time +import numpy as np +import bufr +from pyioda.ioda.Engines.Bufr import Encoder as iodaEncoder +from bufr.encoders.netcdf import Encoder as netcdfEncoder +from wxflow import Logger + +# Initialize Logger +# Get log level from the environment variable, default to 'INFO it not set +log_level = os.getenv('LOG_LEVEL', 'INFO') +logger = Logger('BUFR2IODA_sfcsno.py', level=log_level, colored_log=False) + + +def logging(comm, level, message): + """ + Logs a message to the console or log file, based on the specified logging level. + + This function ensures that logging is only performed by the root process (`rank 0`) + in a distributed computing environment. The function maps the logging level to + appropriate logger methods and defaults to the 'INFO' level if an invalid level is provided. + + Parameters: + comm: object + The communicator object, typically from a distributed computing framework + (e.g., MPI). It must have a `rank()` method to determine the process rank. + level: str + The logging level as a string. Supported levels are: + - 'DEBUG' + - 'INFO' + - 'WARNING' + - 'ERROR' + - 'CRITICAL' + If an invalid level is provided, a warning will be logged, and the level + will default to 'INFO'. + message: str + The message to be logged. + + Behavior: + - Logs messages only on the root process (`comm.rank() == 0`). + - Maps the provided logging level to a method of the logger object. + - Defaults to 'INFO' and logs a warning if an invalid logging level is given. + - Supports standard logging levels for granular control over log verbosity. + + Example: + >>> logging(comm, 'DEBUG', 'This is a debug message.') + >>> logging(comm, 'ERROR', 'An error occurred!') + + Notes: + - Ensure that a global `logger` object is configured before using this function. + - The `comm` object should conform to MPI-like conventions (e.g., `rank()` method). + """ + + if comm.rank() == 0: + # Define a dictionary to map levels to logger methods + log_methods = { + 'DEBUG': logger.debug, + 'INFO': logger.info, + 'WARNING': logger.warning, + 'ERROR': logger.error, + 'CRITICAL': logger.critical, + } + + # Get the appropriate logging method, default to 'INFO' + log_method = log_methods.get(level.upper(), logger.info) + + if log_method == logger.info and level.upper() not in log_methods: + # Log a warning if the level is invalid + logger.warning(f'log level = {level}: not a valid level --> set to INFO') + + # Call the logging method + log_method(message) + + +def _mask_container(container, mask): + + new_container = bufr.DataContainer() + for var_name in container.list(): + var = container.get(var_name) + paths = container.get_paths(var_name) + new_container.add(var_name, var[mask], paths) + + return new_container + + +def _make_description(mapping_path, update=False): + + description = bufr.encoders.Description(mapping_path) + + return description + + +def _make_obs(comm, input_path, mapping_path): + """ + Create the ioda snow depth observations: + - reads state of ground (sogr) and snow depth (snod) + - applys sogr conditions to the missing snod values + - removes the filled/missing snow values and creates the masked container + + Parameters + ---------- + comm: object + The communicator object (e.g., MPI) + input_path: str + The input bufr file + mapping_path: str + The input bufr2ioda mapping file + """ + + # Get container from mapping file first + logging(comm, 'INFO', 'Get container from bufr') + container = bufr.Parser(input_path, mapping_path).parse(comm) + + logging(comm, 'DEBUG', f'container list (original): {container.list()}') + + # Add new/derived data into container + sogr = container.get('variables/groundState') + snod = container.get('variables/totalSnowDepth') + snod[(sogr <= 11.0) & snod.mask] = 0.0 + snod[(sogr == 15.0) & snod.mask] = 0.0 + container.replace('variables/totalSnowDepth', snod) + snod_upd = container.get('variables/totalSnowDepth') + + masked_container = _mask_container(container, (~snod.mask)) + + return masked_container + + +def create_obs_group(input_path, mapping_path, env): + + comm = bufr.mpi.Comm(env["comm_name"]) + + description = _make_description(mapping_path, update=False) + container = _make_obs(comm, input_path, mapping_path) + + # Gather data from all tasks into all tasks. Each task will have the complete record + logging(comm, 'INFO', f'Gather data from all tasks into all tasks') + container.all_gather(comm) + + # Encode the data + logging(comm, 'INFO', f'Encode data') + data = next(iter(iodaEncoder(mapping_path).encode(container).values())) + + logging(comm, 'INFO', f'Return the encoded data') + + return data + + +def create_obs_file(input_path, mapping_path, output_path): + + comm = bufr.mpi.Comm("world") + container = _make_obs(comm, input_path, mapping_path) + container.gather(comm) + + description = _make_description(mapping_path, update=False) + + # Encode the data + if comm.rank() == 0: + netcdfEncoder(description).encode(container, output_path) + + logging(comm, 'INFO', f'Return the encoded data') + + +if __name__ == '__main__': + + start_time = time.time() + + bufr.mpi.App(sys.argv) + comm = bufr.mpi.Comm("world") + + # Required input arguments as positional arguments + parser = argparse.ArgumentParser(description="Convert BUFR to NetCDF using a mapping file.") + parser.add_argument('input', type=str, help='Input BUFR file') + parser.add_argument('mapping', type=str, help='BUFR2IODA Mapping File') + parser.add_argument('output', type=str, help='Output NetCDF file') + + args = parser.parse_args() + mapping = args.mapping + infile = args.input + output = args.output + + create_obs_file(infile, mapping, output) + + end_time = time.time() + running_time = end_time - start_time + logging(comm, 'INFO', f'Total running time: {running_time}') From 5cd21ab3a63010acd06ba4e9cfddff73cc407e39 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 17 Jan 2025 20:24:51 -0500 Subject: [PATCH 11/12] Remove the negative snow depth values. --- parm/snow/jcb-base.yaml.j2 | 1 + ush/ioda/bufr2ioda/bufr_sfcsno.py | 1 + 2 files changed, 2 insertions(+) diff --git a/parm/snow/jcb-base.yaml.j2 b/parm/snow/jcb-base.yaml.j2 index 69e097d74..4f19d6c20 100644 --- a/parm/snow/jcb-base.yaml.j2 +++ b/parm/snow/jcb-base.yaml.j2 @@ -77,6 +77,7 @@ crtm_coefficient_path: "{{ DATA }}/crtm/" # Naming conventions for observational files snow_obsdataroot_path: "{{COMIN_OBS}}" +snow_script_path: "{{snow_script_path}}" snow_obsdatain_path: "{{snow_obsdatain_path}}" snow_obsdatain_prefix: "{{OPREFIX}}" snow_obsdatain_suffix: ".tm00.bufr_d" diff --git a/ush/ioda/bufr2ioda/bufr_sfcsno.py b/ush/ioda/bufr2ioda/bufr_sfcsno.py index 03a9102b4..0eea3c3f3 100755 --- a/ush/ioda/bufr2ioda/bufr_sfcsno.py +++ b/ush/ioda/bufr2ioda/bufr_sfcsno.py @@ -121,6 +121,7 @@ def _make_obs(comm, input_path, mapping_path): snod = container.get('variables/totalSnowDepth') snod[(sogr <= 11.0) & snod.mask] = 0.0 snod[(sogr == 15.0) & snod.mask] = 0.0 + snod.mask = (snod < 0.0) & snod.mask container.replace('variables/totalSnowDepth', snod) snod_upd = container.get('variables/totalSnowDepth') From bb85eb65f499bc3ddecb7bc3dd26bf5128975a40 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 17 Jan 2025 21:43:46 -0500 Subject: [PATCH 12/12] Fix a logic error --- ush/ioda/bufr2ioda/bufr_sfcsno.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr_sfcsno.py b/ush/ioda/bufr2ioda/bufr_sfcsno.py index 0eea3c3f3..cc1a5ce68 100755 --- a/ush/ioda/bufr2ioda/bufr_sfcsno.py +++ b/ush/ioda/bufr2ioda/bufr_sfcsno.py @@ -121,7 +121,7 @@ def _make_obs(comm, input_path, mapping_path): snod = container.get('variables/totalSnowDepth') snod[(sogr <= 11.0) & snod.mask] = 0.0 snod[(sogr == 15.0) & snod.mask] = 0.0 - snod.mask = (snod < 0.0) & snod.mask + snod.mask = (snod < 0.0) | snod.mask container.replace('variables/totalSnowDepth', snod) snod_upd = container.get('variables/totalSnowDepth')