diff --git a/.gitignore b/.gitignore index b5f61746e94..7524452d89f 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ ush/global_chgres_driver.sh ush/global_cycle_driver.sh ush/jediinc2fv3.py ush/imsfv3_scf2ioda.py +ush/bufr_snocvr_snomad.py ush/atparse.bash ush/run_bufr2ioda.py ush/bufr2ioda_insitu* diff --git a/dev/parm/config/gfs/config.esnowanl.j2 b/dev/parm/config/gfs/config.esnowanl.j2 index 2dbf77949b4..786a450858b 100644 --- a/dev/parm/config/gfs/config.esnowanl.j2 +++ b/dev/parm/config/gfs/config.esnowanl.j2 @@ -15,6 +15,9 @@ export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2" export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2" +export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" +export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" + export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/dev/parm/config/gfs/config.snowanl.j2 b/dev/parm/config/gfs/config.snowanl.j2 index 0fd94aa0a72..b88cb429761 100644 --- a/dev/parm/config/gfs/config.snowanl.j2 +++ b/dev/parm/config/gfs/config.snowanl.j2 @@ -15,6 +15,9 @@ export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/apply_incr_nml.j2" export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_config.yaml.j2" export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2" +export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2" +export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py" + export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/dev/ush/load_modules.sh b/dev/ush/load_modules.sh index 18688211340..b222308afbd 100644 --- a/dev/ush/load_modules.sh +++ b/dev/ush/load_modules.sh @@ -151,7 +151,8 @@ case "${MODULE_TYPE}" in # TODO: a better solution should be created for setting paths to package python scripts # shellcheck disable=SC2311 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/" - PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${pyiodaPATH}" + pybufrPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/site-packages/" + PYTHONPATH="${pyiodaPATH}:${pybufrPATH}${PYTHONPATH:+:${PYTHONPATH}}" export PYTHONPATH ;; diff --git a/scripts/exglobal_snow_analysis.py b/scripts/exglobal_snow_analysis.py index 34a84a0538c..9923187d11d 100755 --- a/scripts/exglobal_snow_analysis.py +++ b/scripts/exglobal_snow_analysis.py @@ -23,6 +23,10 @@ # Initialize JEDI 2DVar snow analysis snow_anl.initialize() + # Process SNOCVR and SNOMAD (if applicable) + if snow_anl.task_config.DO_SNOCVR_SNOMAD: + snow_anl.prepare_SNOCVR_SNOMAD() + # Process IMS snow cover (if applicable) if snow_anl.task_config.DO_IMS_SCF: snow_anl.execute('scf_to_ioda') diff --git a/scripts/exglobal_snowens_analysis.py b/scripts/exglobal_snowens_analysis.py index 64197d36488..4ab13d86b96 100755 --- a/scripts/exglobal_snowens_analysis.py +++ b/scripts/exglobal_snowens_analysis.py @@ -29,6 +29,10 @@ # stage ensemble mean backgrounds + # Process SNOCVR and SNOMAD (if applicable) + if snow_ens_anl.task_config.DO_SNOCVR_SNOMAD: + snow_ens_anl.prepare_SNOCVR_SNOMAD() + # Process IMS snow cover (if applicable) if snow_ens_anl.task_config.DO_IMS_SCF: snow_ens_anl.execute('scf_to_ioda') diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6b00f289cf8..f25f3400a1c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6b00f289cf8333133a55bb5ac534a0ddfed74980 +Subproject commit f25f3400a1c68c002400ae0ec456a3177aa596f1 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 109fc346810..243c2df9232 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -266,6 +266,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py" . cd "${HOMEgfs}/ush" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/snow/bufr_snocvr_snomad.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" . declare -a gdasapp_ocn_insitu_profile_platforms=("argo" "bathy" "glider" "marinemammal" "tesac" "xbtctd") for platform in "${gdasapp_ocn_insitu_profile_platforms[@]}"; do diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index c9bb3ad778b..03fdd1312af 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -57,6 +57,14 @@ def __init__(self, config: Dict[str, Any]): else: _DO_IMS_SCF = False + # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing + _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') + _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') + _DO_SNOCVR_SNOMAD = ( + "snocvr_snomad" in self.task_config.observations and + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) + ) + # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( { @@ -65,8 +73,10 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, 'snow_bkg_path': os.path.join('.', 'bkg/'), + 'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'), 'ims_file': _ims_file, 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done + 'DO_SNOCVR_SNOMAD': _DO_SNOCVR_SNOMAD, # Boolean to decide if SNOCVR_SNOMAD processing is done } )) @@ -144,6 +154,69 @@ def finalize(self) -> None: logger.info(f"Saving files to COM") FileHandler(self.task_config.data_out).sync() + @logit(logger) + def prepare_SNOCVR_SNOMAD(self) -> None: + """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis + This includes: + - creating combined SNOCVR and SNOMAD snowdepth data in IODA format. + Parameters + ---------- + self : Analysis + Instance of the SnowAnalysis object + Returns + ---------- + None + """ + + # Read and render the prep_snocvr_snomad.yaml.j2 + logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config) + logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") + + # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2 + logger.info("Copying SNOCVR and SNOMAD obs to DATA") + FileHandler(prep_snocvr_snomad_config.stage).sync() + + # Execute obsBuilder to create the combined snocvr and snomad in IODA format + logger.info("Create the combined snocvr and snomad data in IODA format") + + input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"): + rm_p(output_file) + + logger.info("Link OBSBUILDER into DATA/") + exe_src = self.task_config.OBSBUILDER + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + exe = Executable(exe_dest) + if os.path.exists(input_snocvr): + exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"]) + if os.path.exists(input_snomad): + exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"]) + try: + logger.debug(f"Executing {exe}") + exe() + except OSError: + logger.exception(f"Failed to execute {exe}") + raise + except Exception as err: + logger.exception(f"An error occured during execution of {exe}") + raise WorkflowException(f"An error occured during execution of {exe}") from err + + # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder + # If so, copy to DATA/prep/ + if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"): + logger.warning(f"{output_file} not produced - continuing without it.") + else: + logger.info(f"Copy {output_file} successfully generated") + FileHandler(prep_snocvr_snomad_config.netcdf).sync() + @logit(logger) def add_increments(self) -> None: """Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 7a9d0c7acff..78206324f12 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -54,11 +54,20 @@ def __init__(self, config: Dict[str, Any]): # if 00z, do SCF preprocessing _ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc') + logger.info(f"Checking for IMS file: {_ims_file}") if self.task_config.cyc == 0 and os.path.exists(_ims_file): _DO_IMS_SCF = True else: _DO_IMS_SCF = False + # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing + _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d') + _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d') + _DO_SNOCVR_SNOMAD = ( + "snocvr_snomad" in self.task_config.observations and + (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file)) + ) + # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( { @@ -70,6 +79,7 @@ def __init__(self, config: Dict[str, Any]): 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), 'ims_file': _ims_file, 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done + 'DO_SNOCVR_SNOMAD': _DO_SNOCVR_SNOMAD, # Boolean to decide if SNOCVR_SNOMAD processing is done } )) @@ -148,6 +158,69 @@ def finalize(self) -> None: logger.info(f"Saving files to COM") FileHandler(self.task_config.data_out).sync() + @logit(logger) + def prepare_SNOCVR_SNOMAD(self) -> None: + """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis + This includes: + - creating combined SNOCVR and SNOMAD snowdepth data in IODA format. + Parameters + ---------- + self : Analysis + Instance of the SnowAnalysis object + Returns + ---------- + None + """ + + # Read and render the prep_snocvr_snomad.yaml.j2 + logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}") + prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config) + logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}") + + # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2 + logger.info("Copying SNOCVR and SNOMAD obs to DATA") + FileHandler(prep_snocvr_snomad_config.stage).sync() + + # Execute obsBuilder to create the combined snocvr and snomad in IODA format + logger.info("Create the combined snocvr and snomad data in IODA format") + + input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d' + input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d' + output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc' + if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"): + rm_p(output_file) + + logger.info("Link OBSBUILDER into DATA/") + exe_src = self.task_config.OBSBUILDER + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + exe = Executable(exe_dest) + if os.path.exists(input_snocvr): + exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"]) + exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"]) + if os.path.exists(input_snomad): + exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"]) + try: + logger.debug(f"Executing {exe}") + exe() + except OSError: + logger.exception(f"Failed to execute {exe}") + raise + except Exception as err: + logger.exception(f"An error occured during execution of {exe}") + raise WorkflowException(f"An error occured during execution of {exe}") from err + + # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder + # If so, copy to DATA/prep/ + if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"): + logger.warning(f"{output_file} not produced - continuing without it.") + else: + logger.info(f"Copy {output_file} successfully generated") + FileHandler(prep_snocvr_snomad_config.netcdf).sync() + @logit(logger) def add_increments(self) -> None: """Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds