From 9a7f5834d44b23e55eaf1d3129df4ca64d8797f9 Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Fri, 3 Oct 2025 21:15:00 -0400 Subject: [PATCH 01/13] Add snomad and use the combined snocvr and snomad snow depth data. --- .gitignore | 1 + dev/parm/config/gfs/config.snowanl.j2 | 4 ++ scripts/exglobal_snow_analysis.py | 4 ++ sorc/link_workflow.sh | 1 + ush/python/pygfs/task/snow_analysis.py | 90 +++++++++++++++++++++++++- 5 files changed, 98 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e0e68a8f2ec..e83b5eaad7c 100644 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,7 @@ ush/global_chgres_driver.sh ush/global_cycle_driver.sh ush/jediinc2fv3.py ush/imsfv3_scf2ioda.py +ush/bufr_snocvr_snomad.py ush/atparse.bash ush/run_bufr2ioda.py ush/bufr2ioda_insitu* diff --git a/dev/parm/config/gfs/config.snowanl.j2 b/dev/parm/config/gfs/config.snowanl.j2 index 83f1a9ae78a..a83a21496bd 100644 --- a/dev/parm/config/gfs/config.snowanl.j2 +++ b/dev/parm/config/gfs/config.snowanl.j2 @@ -19,6 +19,10 @@ export STAGE_BERROR_YAML="${PARMgfs}/gdas/snow/snow_stage_berror.yaml.j2" export STAGE_GTS_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" export STAGE_IMS_SCF2IODA_YAML="${PARMgfs}/gdas/snow/snow_stage_ims_scf2ioda.yaml.j2" +export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" +export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" +export PYTHONPATH=${HOMEgfs}/lib/python3.11/site-packages:${PYTHONPATH} + export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/scripts/exglobal_snow_analysis.py b/scripts/exglobal_snow_analysis.py index 66721913737..8a9145371b4 100755 --- a/scripts/exglobal_snow_analysis.py +++ b/scripts/exglobal_snow_analysis.py @@ -23,6 +23,10 @@ # Initialize JEDI 2DVar snow analysis snow_anl.initialize() + # Process SNOCVR and SNOMAD (if applicable) + if snow_anl.task_config.DO_SNOCVR_SNOMAD == True: + snow_anl.prepare_SNOCVR_SNOMAD() + # Process IMS snow cover (if applicable) if snow_anl.task_config.cyc == 0: snow_anl.execute('scf_to_ioda') diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f57ea472dde..b511a21af61 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -256,6 +256,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py" . cd "${HOMEgfs}/ush" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/snow/bufr_snocvr_snomad.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" . declare -a gdasapp_ocn_insitu_profile_platforms=("argo" "bathy" "glider" "marinemammal" "tesac" "xbtctd") for platform in "${gdasapp_ocn_insitu_profile_platforms[@]}"; do diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9f7b2a83ed5..8ca5877c5fe 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -69,6 +69,7 @@ def __init__(self, config: Dict[str, Any]): 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'), 'snow_obsdatain_path': os.path.join(self.task_config.DATA, 'obs'), 'snow_obsdataout_path': os.path.join(self.task_config.DATA, 'diags'), 'snow_bkg_path': os.path.join('.', 'bkg/'), @@ -83,6 +84,82 @@ def __init__(self, config: Dict[str, Any]): expected_keys = ['scf_to_ioda', 'snowanlvar'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + # Boolean to decide if SNOCVR and SNOMAD processing is done + self.task_config.DO_SNOCVR_SNOMAD = False + + @logit(logger) + def prepare_SNOCVR_SNOMAD(self) -> None: + """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis + + This includes: + - creating combined SNOCVR and SNOMAD snowdepth data in IODA format. + + Parameters + ---------- + self : Analysis + Instance of the SnowAnalysis object + + Returns + ---------- + None + """ + + # create a temporary dict of all keys needed in this method + localconf = AttrDict() + keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX','snow_prepobs_path'] + for key in keys: + localconf[key] = self.task_config[key] + + # Read and render the prep_snocvr_snomad.yaml.j2 + logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, localconf) + logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") + + # define these locations in gdas/snow/prep/prep_ghcn.yaml.j2 + logger.info("Copying SNOCVR and SNOMAD obs to DATA") + FileHandler(prep_snocvr_snomad_config.stage).sync() + + # Execute obsBuilder to create the combined snocvr and snomad in IODA format + logger.info("Create the combined snocvr and snomad data in IODA format") + + input_snocvr = f'{localconf.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{localconf.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{localconf.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(localconf.DATA, output_file)}"): + rm_p(output_file) + + logger.info("Link OBSBUILDER into DATA/") + exe_src = self.task_config.OBSBUILDER + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + exe = Executable(exe_dest) + if os.path.exists(input_snocvr): + exe.add_default_arg(["--input_snocvr", f"{os.path.join(localconf.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(localconf.DATA, output_file)}"]) + if os.path.exists(input_snomad): + exe.add_default_arg(["--input_snomad", f"{os.path.join(localconf.DATA, input_snomad)}"]) + try: + logger.debug(f"Executing {exe}") + exe() + except OSError: + logger.exception(f"Failed to execute {exe}") + raise + except Exception as err: + logger.exception(f"An error occured during execution of {exe}") + raise WorkflowException(f"An error occured during execution of {exe}") from err + + # Ensure the IODA snow depth GHCN file is produced by the IODA converter + # If so, copy to DATA/obs/ + if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): + logger.exception(f"{self.task_config.OBSBUILDER} failed to produce {output_file}") + raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}") + else: + logger.info(f"Copy {output_file} successfully generated") + FileHandler(prep_snocvr_snomad_config.netcdf).sync() + @logit(logger) def initialize(self) -> None: """Initialize a global snow analysis @@ -142,8 +219,15 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() + # Check if SNOMAD file exists + snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') + snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') + if os.path.exists(snocvr_file) or os.path.exists(snomad_file): + self.task_config.DO_SNOCVR_SNOMAD = True + # if 00z, do SCF preprocessing - if self.task_config.cyc == 0: + infile = os.path.join(self.task_config.COMIN_OBS,f'{self.task_config.OPREFIX}imssnow96.asc') + if self.task_config.cyc == 0 and os.path.exists(infile): ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) FileHandler(ims_scf_to_ioda_staging_dict).sync() self.jedi_dict['scf_to_ioda'].initialize(self.task_config) @@ -168,7 +252,9 @@ def execute(self, jedi_dict_key: str) -> None: None """ - self.jedi_dict[jedi_dict_key].execute() + infile = os.path.join(self.task_config.DATA,f'{jedi_dict_key}.yaml') + if os.path.exists(infile): + self.jedi_dict[jedi_dict_key].execute() @logit(logger) def finalize(self) -> None: From 87b0b2ae0aca535817ead5495c26e8b2f46c039f Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Sat, 4 Oct 2025 14:01:35 -0400 Subject: [PATCH 02/13] Fix pynorms errors --- scripts/exglobal_snow_analysis.py | 2 +- ush/python/pygfs/task/snow_analysis.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_snow_analysis.py b/scripts/exglobal_snow_analysis.py index 8a9145371b4..720c3aafb2c 100755 --- a/scripts/exglobal_snow_analysis.py +++ b/scripts/exglobal_snow_analysis.py @@ -24,7 +24,7 @@ snow_anl.initialize() # Process SNOCVR and SNOMAD (if applicable) - if snow_anl.task_config.DO_SNOCVR_SNOMAD == True: + if snow_anl.task_config.DO_SNOCVR_SNOMAD: snow_anl.prepare_SNOCVR_SNOMAD() # Process IMS snow cover (if applicable) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 8ca5877c5fe..cea130a7c20 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -106,7 +106,7 @@ def prepare_SNOCVR_SNOMAD(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() - keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX','snow_prepobs_path'] + keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX', 'snow_prepobs_path'] for key in keys: localconf[key] = self.task_config[key] @@ -226,7 +226,7 @@ def initialize(self) -> None: self.task_config.DO_SNOCVR_SNOMAD = True # if 00z, do SCF preprocessing - infile = os.path.join(self.task_config.COMIN_OBS,f'{self.task_config.OPREFIX}imssnow96.asc') + infile = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc') if self.task_config.cyc == 0 and os.path.exists(infile): ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) FileHandler(ims_scf_to_ioda_staging_dict).sync() @@ -252,7 +252,7 @@ def execute(self, jedi_dict_key: str) -> None: None """ - infile = os.path.join(self.task_config.DATA,f'{jedi_dict_key}.yaml') + infile = os.path.join(self.task_config.DATA, f'{jedi_dict_key}.yaml') if os.path.exists(infile): self.jedi_dict[jedi_dict_key].execute() From 92e17e0caf6760021327ea6172e67b7bc03af426 Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Sun, 5 Oct 2025 18:47:45 -0400 Subject: [PATCH 03/13] Merge to develop --- ush/python/pygfs/task/snow_analysis.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index cb372008e11..299de86e485 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -229,8 +229,7 @@ def initialize(self) -> None: self.task_config.DO_SNOCVR_SNOMAD = True # if 00z, do SCF preprocessing - infile = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc') - if self.task_config.cyc == 0 and os.path.exists(infile): + if self.task_config.cyc == 0: ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) FileHandler(ims_scf_to_ioda_staging_dict).sync() self.jedi_dict['scf_to_ioda'].initialize(self.task_config) @@ -259,9 +258,7 @@ def execute(self, jedi_dict_key: str) -> None: None """ - infile = os.path.join(self.task_config.DATA, f'{jedi_dict_key}.yaml') - if os.path.exists(infile): - self.jedi_dict[jedi_dict_key].execute() + self.jedi_dict[jedi_dict_key].execute() @logit(logger) def finalize(self) -> None: From 02c144c0faef4121f733d24e2f189a20ba5b76ad Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Mon, 6 Oct 2025 17:34:33 -0400 Subject: [PATCH 04/13] Add the same changes to esnowanl and move the PYTHONPATH to dev/ush --- dev/parm/config/gfs/config.snowanl.j2 | 1 - dev/ush/load_ufsda_modules.sh | 3 +- scripts/exglobal_snowens_analysis.py | 4 ++ ush/python/pygfs/task/snow_analysis.py | 2 +- ush/python/pygfs/task/snowens_analysis.py | 80 +++++++++++++++++++++++ 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/dev/parm/config/gfs/config.snowanl.j2 b/dev/parm/config/gfs/config.snowanl.j2 index a83a21496bd..7dfda36a355 100644 --- a/dev/parm/config/gfs/config.snowanl.j2 +++ b/dev/parm/config/gfs/config.snowanl.j2 @@ -21,7 +21,6 @@ export STAGE_IMS_SCF2IODA_YAML="${PARMgfs}/gdas/snow/snow_stage_ims_scf2ioda.yam export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" -export PYTHONPATH=${HOMEgfs}/lib/python3.11/site-packages:${PYTHONPATH} export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/dev/ush/load_ufsda_modules.sh b/dev/ush/load_ufsda_modules.sh index 63314002c46..1b9d035beb7 100755 --- a/dev/ush/load_ufsda_modules.sh +++ b/dev/ush/load_ufsda_modules.sh @@ -102,7 +102,8 @@ fi # TODO: a better solution should be created for setting paths to package python scripts # shellcheck disable=SC2311 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${pyiodaPATH}" +pybufrPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/site-packages/" +PYTHONPATH="${pyiodaPATH}:${pybufrPATH}${PYTHONPATH:+:${PYTHONPATH}}" export PYTHONPATH # Restore stack soft limit: diff --git a/scripts/exglobal_snowens_analysis.py b/scripts/exglobal_snowens_analysis.py index 64197d36488..ea1759c72f4 100755 --- a/scripts/exglobal_snowens_analysis.py +++ b/scripts/exglobal_snowens_analysis.py @@ -29,6 +29,10 @@ # stage ensemble mean backgrounds + # Process SNOCVR and SNOMAD (if applicable) + if snow_anl.task_config.DO_SNOCVR_SNOMAD: + snow_anl.prepare_SNOCVR_SNOMAD() + # Process IMS snow cover (if applicable) if snow_ens_anl.task_config.DO_IMS_SCF: snow_ens_anl.execute('scf_to_ioda') diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 299de86e485..532507a63dc 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -222,7 +222,7 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() - # Check if SNOMAD file exists + # Check if SNOCVR or SNOMAD file exists snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') if os.path.exists(snocvr_file) or os.path.exists(snomad_file): diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 9a94a55c9e5..43d1a1f70ae 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -70,6 +70,7 @@ def __init__(self, config: Dict[str, Any]): 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'), 'snow_obsdatain_path': os.path.join(self.task_config.DATA, 'obs'), 'snow_obsdataout_path': os.path.join(self.task_config.DATA, 'diags'), 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), @@ -87,6 +88,79 @@ def __init__(self, config: Dict[str, Any]): # Boolean to decide if IMS snow cover processing is done self.task_config.DO_IMS_SCF = False + # Boolean to decide if SNOCVR and SNOMAD processing is done + self.task_config.DO_SNOCVR_SNOMAD = False + + @logit(logger) + def prepare_SNOCVR_SNOMAD(self) -> None: + """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis + This includes: + - creating combined SNOCVR and SNOMAD snowdepth data in IODA format. + Parameters + ---------- + self : Analysis + Instance of the SnowAnalysis object + Returns + ---------- + None + """ + + # create a temporary dict of all keys needed in this method + localconf = AttrDict() + keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX', 'snow_prepobs_path'] + for key in keys: + localconf[key] = self.task_config[key] + + # Read and render the prep_snocvr_snomad.yaml.j2 + logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, localconf) + logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") + + # define these locations in gdas/snow/prep/prep_ghcn.yaml.j2 + logger.info("Copying SNOCVR and SNOMAD obs to DATA") + FileHandler(prep_snocvr_snomad_config.stage).sync() + + # Execute obsBuilder to create the combined snocvr and snomad in IODA format + logger.info("Create the combined snocvr and snomad data in IODA format") + + input_snocvr = f'{localconf.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{localconf.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{localconf.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(localconf.DATA, output_file)}"): + rm_p(output_file) + + logger.info("Link OBSBUILDER into DATA/") + exe_src = self.task_config.OBSBUILDER + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + exe = Executable(exe_dest) + if os.path.exists(input_snocvr): + exe.add_default_arg(["--input_snocvr", f"{os.path.join(localconf.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(localconf.DATA, output_file)}"]) + if os.path.exists(input_snomad): + exe.add_default_arg(["--input_snomad", f"{os.path.join(localconf.DATA, input_snomad)}"]) + try: + logger.debug(f"Executing {exe}") + exe() + except OSError: + logger.exception(f"Failed to execute {exe}") + raise + except Exception as err: + logger.exception(f"An error occured during execution of {exe}") + raise WorkflowException(f"An error occured during execution of {exe}") from err + + # Ensure the IODA snow depth GHCN file is produced by the IODA converter + # If so, copy to DATA/obs/ + if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): + logger.exception(f"{self.task_config.OBSBUILDER} failed to produce {output_file}") + raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}") + else: + logger.info(f"Copy {output_file} successfully generated") + FileHandler(prep_snocvr_snomad_config.netcdf).sync() + @logit(logger) def initialize(self) -> None: """Initialize a global snow ensemble analysis @@ -163,6 +237,12 @@ def initialize(self) -> None: ] FileHandler({'mkdir': newdirs}).sync() + # Check if SNOCVR or SNOMAD file exists + snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') + snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') + if os.path.exists(snocvr_file) or os.path.exists(snomad_file): + self.task_config.DO_SNOCVR_SNOMAD = True + # if 00z, do SCF preprocessing if self.task_config.cyc == 0: ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) From 9bb1bbb3a304a64c71b518fdfeb82011cc18fb7a Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Mon, 6 Oct 2025 17:45:19 -0400 Subject: [PATCH 05/13] Update the GDASApp repo --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 77f286df1e6..cdb9122d5eb 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 77f286df1e640551ec23b0909adf71fd5f2c57e5 +Subproject commit cdb9122d5eb148de01a4d79f9294b77cb32ad903 From 3940cf2d715e13ae859b258041bd0183ee100e14 Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Mon, 6 Oct 2025 19:37:30 -0400 Subject: [PATCH 06/13] Update the config.esnowanl --- dev/parm/config/gfs/config.esnowanl.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/parm/config/gfs/config.esnowanl.j2 b/dev/parm/config/gfs/config.esnowanl.j2 index 4da6cf78473..820b51e4c23 100644 --- a/dev/parm/config/gfs/config.esnowanl.j2 +++ b/dev/parm/config/gfs/config.esnowanl.j2 @@ -17,6 +17,9 @@ export STAGE_IMS_SCF2IODA_YAML="${PARMgfs}/gdas/snow/snow_stage_ims_scf2ioda.yam export STAGE_GTS_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" export SAVE_YAML="${PARMgfs}/gdas/snow/snow_ens_save.yaml.j2" +export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" +export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" + # Name of the executable that applies increment to bkg and its namelist template export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2" From 54e24df7e627e4febe27daa51588d32775794b28 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 7 Oct 2025 10:08:21 -0400 Subject: [PATCH 07/13] Update snow_analysis.py Update warning and ensure the process continues --- ush/python/pygfs/task/snow_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 532507a63dc..f15e7894c0d 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -154,11 +154,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: logger.exception(f"An error occured during execution of {exe}") raise WorkflowException(f"An error occured during execution of {exe}") from err - # Ensure the IODA snow depth GHCN file is produced by the IODA converter - # If so, copy to DATA/obs/ + # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder + # If so, copy to DATA/prep/ if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): - logger.exception(f"{self.task_config.OBSBUILDER} failed to produce {output_file}") - raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}") + logger.warning(f"{output_file} not produced - continuing without it.") else: logger.info(f"Copy {output_file} successfully generated") FileHandler(prep_snocvr_snomad_config.netcdf).sync() From 63a8620e31561d07bbb67674b7106928d13f9bf0 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 7 Oct 2025 10:10:30 -0400 Subject: [PATCH 08/13] Update snowens_analysis.py Change to warning and ensure the process continues --- ush/python/pygfs/task/snowens_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 43d1a1f70ae..c01c5d53eba 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -152,11 +152,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: logger.exception(f"An error occured during execution of {exe}") raise WorkflowException(f"An error occured during execution of {exe}") from err - # Ensure the IODA snow depth GHCN file is produced by the IODA converter - # If so, copy to DATA/obs/ + # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder + # If so, copy to DATA/prep/ if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): - logger.exception(f"{self.task_config.OBSBUILDER} failed to produce {output_file}") - raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}") + logger.warning(f"{output_file} not produced - continuing without it.") else: logger.info(f"Copy {output_file} successfully generated") FileHandler(prep_snocvr_snomad_config.netcdf).sync() From 9522fca029edef5e6064e9871a8f2671008c898c Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Fri, 10 Oct 2025 12:55:10 -0400 Subject: [PATCH 09/13] Add the SNOCVR_SNOMAD to esnowanl --- dev/parm/config/gfs/config.esnowanl.j2 | 6 +++--- dev/ush/load_modules.sh | 3 ++- scripts/exglobal_snowens_analysis.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/parm/config/gfs/config.esnowanl.j2 b/dev/parm/config/gfs/config.esnowanl.j2 index c94d476507c..786a450858b 100644 --- a/dev/parm/config/gfs/config.esnowanl.j2 +++ b/dev/parm/config/gfs/config.esnowanl.j2 @@ -11,13 +11,13 @@ source "${EXPDIR}/config.resources" esnowanl export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_config.yaml.j2" export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2" -export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" -export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" - # Name of the executable that applies increment to bkg and its namelist template export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2" +export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" +export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" + export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/dev/ush/load_modules.sh b/dev/ush/load_modules.sh index 18688211340..b222308afbd 100644 --- a/dev/ush/load_modules.sh +++ b/dev/ush/load_modules.sh @@ -151,7 +151,8 @@ case "${MODULE_TYPE}" in # TODO: a better solution should be created for setting paths to package python scripts # shellcheck disable=SC2311 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/" - PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${pyiodaPATH}" + pybufrPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/site-packages/" + PYTHONPATH="${pyiodaPATH}:${pybufrPATH}${PYTHONPATH:+:${PYTHONPATH}}" export PYTHONPATH ;; diff --git a/scripts/exglobal_snowens_analysis.py b/scripts/exglobal_snowens_analysis.py index ea1759c72f4..4ab13d86b96 100755 --- a/scripts/exglobal_snowens_analysis.py +++ b/scripts/exglobal_snowens_analysis.py @@ -30,8 +30,8 @@ # stage ensemble mean backgrounds # Process SNOCVR and SNOMAD (if applicable) - if snow_anl.task_config.DO_SNOCVR_SNOMAD: - snow_anl.prepare_SNOCVR_SNOMAD() + if snow_ens_anl.task_config.DO_SNOCVR_SNOMAD: + snow_ens_anl.prepare_SNOCVR_SNOMAD() # Process IMS snow cover (if applicable) if snow_ens_anl.task_config.DO_IMS_SCF: From abc3acf771e5f40828c6c77b43073e17bdfad608 Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Fri, 10 Oct 2025 15:18:09 -0400 Subject: [PATCH 10/13] Adress reviewer's comments --- ush/python/pygfs/task/snow_analysis.py | 32 +++++++++-------------- ush/python/pygfs/task/snowens_analysis.py | 32 +++++++++-------------- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 3729e7b8f40..1fd9ee378dc 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -60,10 +60,10 @@ def __init__(self, config: Dict[str, Any]): # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') - if os.path.exists(_snocvr_file) or os.path.exists(_snomad_file): - _DO_SNOCVR_SNOMAD = True - else: - _DO_SNOCVR_SNOMAD = False + _DO_SNOCVR_SNOMAD = any( + ob == "snocvr_snomad" and (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) + for ob in self.task_config.observations + ) # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( @@ -168,15 +168,9 @@ def prepare_SNOCVR_SNOMAD(self) -> None: None """ - # create a temporary dict of all keys needed in this method - localconf = AttrDict() - keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX', 'snow_prepobs_path'] - for key in keys: - localconf[key] = self.task_config[key] - # Read and render the prep_snocvr_snomad.yaml.j2 logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") - prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, localconf) + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config) logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2 @@ -186,10 +180,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: # Execute obsBuilder to create the combined snocvr and snomad in IODA format logger.info("Create the combined snocvr and snomad data in IODA format") - input_snocvr = f'{localconf.OPREFIX}snocvr.tm00.bufr_d' - input_snomad = f'{localconf.OPREFIX}snomad.tm00.bufr_d' - output_file = f'{localconf.OPREFIX}snocvr_snomad.tm00.nc' - if os.path.exists(f"{os.path.join(localconf.DATA, output_file)}"): + input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"): rm_p(output_file) logger.info("Link OBSBUILDER into DATA/") @@ -201,10 +195,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: exe = Executable(exe_dest) if os.path.exists(input_snocvr): - exe.add_default_arg(["--input_snocvr", f"{os.path.join(localconf.DATA, input_snocvr)}"]) - exe.add_default_arg(["--output", f"{os.path.join(localconf.DATA, output_file)}"]) + exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"]) if os.path.exists(input_snomad): - exe.add_default_arg(["--input_snomad", f"{os.path.join(localconf.DATA, input_snomad)}"]) + exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"]) try: logger.debug(f"Executing {exe}") exe() @@ -217,7 +211,7 @@ def prepare_SNOCVR_SNOMAD(self) -> None: # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder # If so, copy to DATA/prep/ - if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): + if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"): logger.warning(f"{output_file} not produced - continuing without it.") else: logger.info(f"Copy {output_file} successfully generated") diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index a40d659cb12..4d879250fc6 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -63,10 +63,10 @@ def __init__(self, config: Dict[str, Any]): # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') - if os.path.exists(_snocvr_file) or os.path.exists(_snomad_file): - _DO_SNOCVR_SNOMAD = True - else: - _DO_SNOCVR_SNOMAD = False + _DO_SNOCVR_SNOMAD = any( + ob == "snocvr_snomad" and (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) + for ob in self.task_config.observations + ) # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( @@ -172,15 +172,9 @@ def prepare_SNOCVR_SNOMAD(self) -> None: None """ - # create a temporary dict of all keys needed in this method - localconf = AttrDict() - keys = ['DATA', 'PARMgfs', 'COMIN_OBS', 'OPREFIX', 'snow_prepobs_path'] - for key in keys: - localconf[key] = self.task_config[key] - # Read and render the prep_snocvr_snomad.yaml.j2 logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") - prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, localconf) + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config) logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2 @@ -190,10 +184,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: # Execute obsBuilder to create the combined snocvr and snomad in IODA format logger.info("Create the combined snocvr and snomad data in IODA format") - input_snocvr = f'{localconf.OPREFIX}snocvr.tm00.bufr_d' - input_snomad = f'{localconf.OPREFIX}snomad.tm00.bufr_d' - output_file = f'{localconf.OPREFIX}snocvr_snomad.tm00.nc' - if os.path.exists(f"{os.path.join(localconf.DATA, output_file)}"): + input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"): rm_p(output_file) logger.info("Link OBSBUILDER into DATA/") @@ -205,10 +199,10 @@ def prepare_SNOCVR_SNOMAD(self) -> None: exe = Executable(exe_dest) if os.path.exists(input_snocvr): - exe.add_default_arg(["--input_snocvr", f"{os.path.join(localconf.DATA, input_snocvr)}"]) - exe.add_default_arg(["--output", f"{os.path.join(localconf.DATA, output_file)}"]) + exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"]) if os.path.exists(input_snomad): - exe.add_default_arg(["--input_snomad", f"{os.path.join(localconf.DATA, input_snomad)}"]) + exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"]) try: logger.debug(f"Executing {exe}") exe() @@ -221,7 +215,7 @@ def prepare_SNOCVR_SNOMAD(self) -> None: # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder # If so, copy to DATA/prep/ - if not os.path.isfile(f"{os.path.join(localconf.DATA, output_file)}"): + if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"): logger.warning(f"{output_file} not produced - continuing without it.") else: logger.info(f"Copy {output_file} successfully generated") From 5f8653f4bd693ed1a27413acd028d6b96e8de6ce Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Fri, 10 Oct 2025 15:35:41 -0400 Subject: [PATCH 11/13] Update the GDASApp latest commit hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6b00f289cf8..f25f3400a1c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6b00f289cf8333133a55bb5ac534a0ddfed74980 +Subproject commit f25f3400a1c68c002400ae0ec456a3177aa596f1 From d00f468633bbf0d3f50a2af6a2992f25310aa27f Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Tue, 14 Oct 2025 11:17:13 -0400 Subject: [PATCH 12/13] Modify for better readability and maintainability. --- ush/python/pygfs/task/snow_analysis.py | 6 +++--- ush/python/pygfs/task/snowens_analysis.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 1fd9ee378dc..0cc893e20c5 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -60,9 +60,9 @@ def __init__(self, config: Dict[str, Any]): # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') - _DO_SNOCVR_SNOMAD = any( - ob == "snocvr_snomad" and (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) - for ob in self.task_config.observations + _DO_SNOCVR_SNOMAD = ( + "snocvr_snomad" in self.task_config.observations and + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file) ) # Extend task_config with variables repeatedly used across this class diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 4d879250fc6..788bc5aecea 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -63,9 +63,9 @@ def __init__(self, config: Dict[str, Any]): # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') - _DO_SNOCVR_SNOMAD = any( - ob == "snocvr_snomad" and (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) - for ob in self.task_config.observations + _DO_SNOCVR_SNOMAD = ( + "snocvr_snomad" in self.task_config.observations and + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file) ) # Extend task_config with variables repeatedly used across this class From 4854a37501cf17fd9008e74e13155db2414a2876 Mon Sep 17 00:00:00 2001 From: jiaruidong2017 Date: Tue, 14 Oct 2025 11:25:27 -0400 Subject: [PATCH 13/13] Fixed the pynorm errors --- ush/python/pygfs/task/snow_analysis.py | 2 +- ush/python/pygfs/task/snowens_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 0cc893e20c5..03fdd1312af 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -62,7 +62,7 @@ def __init__(self, config: Dict[str, Any]): _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') _DO_SNOCVR_SNOMAD = ( "snocvr_snomad" in self.task_config.observations and - (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file) + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) ) # Extend task_config with variables repeatedly used across this class diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 788bc5aecea..78206324f12 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -65,7 +65,7 @@ def __init__(self, config: Dict[str, Any]): _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') _DO_SNOCVR_SNOMAD = ( "snocvr_snomad" in self.task_config.observations and - (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file) + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) ) # Extend task_config with variables repeatedly used across this class