Add functionality to assimilate the new snow observations

.gitignore

@@ -171,6 +171,7 @@ ush/global_chgres_driver.sh
 ush/global_cycle_driver.sh
 ush/jediinc2fv3.py
 ush/imsfv3_scf2ioda.py
+ush/bufr_snocvr_snomad.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
 ush/bufr2ioda_insitu*

dev/parm/config/gfs/config.esnowanl.j2

@@ -15,6 +15,9 @@ export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2"
 export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x"
 export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2"
 
+export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2"
+export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py"
+
 export io_layout_x="{{ IO_LAYOUT_X }}"
 export io_layout_y="{{ IO_LAYOUT_Y }}"
 

dev/parm/config/gfs/config.snowanl.j2

@@ -15,6 +15,9 @@ export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/apply_incr_nml.j2"
 export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_config.yaml.j2"
 export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2"
 
+export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2"
+export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py"
+
 export io_layout_x="{{ IO_LAYOUT_X }}"
 export io_layout_y="{{ IO_LAYOUT_Y }}"
 

dev/ush/load_modules.sh

@@ -151,7 +151,8 @@ case "${MODULE_TYPE}" in
     # TODO: a better solution should be created for setting paths to package python scripts
     # shellcheck disable=SC2311
     pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/"
-    PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${pyiodaPATH}"
+    pybufrPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python${PYTHON_VERSION}/site-packages/"
+    PYTHONPATH="${pyiodaPATH}:${pybufrPATH}${PYTHONPATH:+:${PYTHONPATH}}"
     export PYTHONPATH
     ;;
 

scripts/exglobal_snow_analysis.py

@@ -23,6 +23,10 @@
     # Initialize JEDI 2DVar snow analysis
     snow_anl.initialize()
 
+    # Process SNOCVR and SNOMAD (if applicable)
+    if snow_anl.task_config.DO_SNOCVR_SNOMAD:
+        snow_anl.prepare_SNOCVR_SNOMAD()
+
     # Process IMS snow cover (if applicable)
     if snow_anl.task_config.DO_IMS_SCF:
         snow_anl.execute('scf_to_ioda')

scripts/exglobal_snowens_analysis.py

@@ -29,6 +29,10 @@
 
     # stage ensemble mean backgrounds
 
+    # Process SNOCVR and SNOMAD (if applicable)
+    if snow_ens_anl.task_config.DO_SNOCVR_SNOMAD:
+        snow_ens_anl.prepare_SNOCVR_SNOMAD()
+
     # Process IMS snow cover (if applicable)
     if snow_ens_anl.task_config.DO_IMS_SCF:
         snow_ens_anl.execute('scf_to_ioda')

sorc/link_workflow.sh

@@ -266,6 +266,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/gen_bufr2ioda_yaml.py"    .
   cd "${HOMEgfs}/ush" || exit 1
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py"    .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/snow/bufr_snocvr_snomad.py"         .
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py"           .
   declare -a gdasapp_ocn_insitu_profile_platforms=("argo" "bathy" "glider" "marinemammal" "tesac" "xbtctd")
   for platform in "${gdasapp_ocn_insitu_profile_platforms[@]}"; do

ush/python/pygfs/task/snow_analysis.py

@@ -57,6 +57,14 @@ def __init__(self, config: Dict[str, Any]):
         else:
             _DO_IMS_SCF = False
 
+        # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing
+        _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d')
+        _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d')
+        _DO_SNOCVR_SNOMAD = (
+            "snocvr_snomad" in self.task_config.observations and
+            (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file))
+        )
+
         # Extend task_config with variables repeatedly used across this class
         self.task_config.update(AttrDict(
             {
@@ -65,8 +73,10 @@ def __init__(self, config: Dict[str, Any]):
                 'npz_ges': self.task_config.LEVS - 1,
                 'npz': self.task_config.LEVS - 1,
                 'snow_bkg_path': os.path.join('.', 'bkg/'),
+                'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'),
                 'ims_file': _ims_file,
                 'DO_IMS_SCF': _DO_IMS_SCF,  # Boolean to decide if IMS snow cover processing is done
+                'DO_SNOCVR_SNOMAD': _DO_SNOCVR_SNOMAD,  # Boolean to decide if SNOCVR_SNOMAD processing is done
             }
         ))
 
@@ -144,6 +154,69 @@ def finalize(self) -> None:
         logger.info(f"Saving files to COM")
         FileHandler(self.task_config.data_out).sync()
 
+    @logit(logger)
+    def prepare_SNOCVR_SNOMAD(self) -> None:
+        """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis
+        This includes:
+        - creating combined SNOCVR and SNOMAD snowdepth data in IODA format.
+        Parameters
+        ----------
+        self : Analysis
+            Instance of the SnowAnalysis object
+        Returns
+        ----------
+        None
+        """
+
+        # Read and render the prep_snocvr_snomad.yaml.j2
+        logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}")
+        prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config)
+        logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}")
+
+        # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2
+        logger.info("Copying SNOCVR and SNOMAD obs to DATA")
+        FileHandler(prep_snocvr_snomad_config.stage).sync()
+
+        # Execute obsBuilder to create the combined snocvr and snomad in IODA format
+        logger.info("Create the combined snocvr and snomad data in IODA format")
+
+        input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d'
+        input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d'
+        output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc'
+        if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"):
+            rm_p(output_file)
+
+        logger.info("Link OBSBUILDER into DATA/")
+        exe_src = self.task_config.OBSBUILDER
+        exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src))
+        if os.path.exists(exe_dest):
+            rm_p(exe_dest)
+        os.symlink(exe_src, exe_dest)
+
+        exe = Executable(exe_dest)
+        if os.path.exists(input_snocvr):
+            exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"])
+        exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"])
+        if os.path.exists(input_snomad):
+            exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"])
+        try:
+            logger.debug(f"Executing {exe}")
+            exe()
+        except OSError:
+            logger.exception(f"Failed to execute {exe}")
+            raise
+        except Exception as err:
+            logger.exception(f"An error occured during execution of {exe}")
+            raise WorkflowException(f"An error occured during execution of {exe}") from err
+
+        # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder
+        # If so, copy to DATA/prep/
+        if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"):
+            logger.warning(f"{output_file} not produced - continuing without it.")
+        else:
+            logger.info(f"Copy {output_file} successfully generated")
+            FileHandler(prep_snocvr_snomad_config.netcdf).sync()
+
     @logit(logger)
     def add_increments(self) -> None:
         """Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds

ush/python/pygfs/task/snowens_analysis.py

@@ -54,11 +54,20 @@ def __init__(self, config: Dict[str, Any]):
 
         # if 00z, do SCF preprocessing
         _ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc')
+        logger.info(f"Checking for IMS file: {_ims_file}")
         if self.task_config.cyc == 0 and os.path.exists(_ims_file):
             _DO_IMS_SCF = True
         else:
             _DO_IMS_SCF = False
 
+        # Check if SNOCVR or SNOMAD file exists, do SNOCVR_SNOMAD preprocessing
+        _snocvr_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d')
+        _snomad_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}snomad.tm00.bufr_d')
+        _DO_SNOCVR_SNOMAD = (
+            "snocvr_snomad" in self.task_config.observations and
+            (os.path.exists(_snocvr_file) or os.path.exists(_snomad_file))
+        )
+
         # Extend task_config with variables repeatedly used across this class
         self.task_config.update(AttrDict(
             {
@@ -70,6 +79,7 @@ def __init__(self, config: Dict[str, Any]):
                 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'),
                 'ims_file': _ims_file,
                 'DO_IMS_SCF': _DO_IMS_SCF,  # Boolean to decide if IMS snow cover processing is done
+                'DO_SNOCVR_SNOMAD': _DO_SNOCVR_SNOMAD,  # Boolean to decide if SNOCVR_SNOMAD processing is done
             }
         ))
 
@@ -148,6 +158,69 @@ def finalize(self) -> None:
         logger.info(f"Saving files to COM")
         FileHandler(self.task_config.data_out).sync()
 
+    @logit(logger)
+    def prepare_SNOCVR_SNOMAD(self) -> None:
+        """Prepare the combined SNOCVR and SNOMAD data for a global snow analysis
+        This includes:
+        - creating combined SNOCVR and SNOMAD snowdepth data in IODA format.
+        Parameters
+        ----------
+        self : Analysis
+            Instance of the SnowAnalysis object
+        Returns
+        ----------
+        None
+        """
+
+        # Read and render the prep_snocvr_snomad.yaml.j2
+        logger.info(f"Reading {self.task_config.PREP_SNOCVR_SNOMAD_YAML}")
+        prep_snocvr_snomad_config = parse_j2yaml(self.task_config.PREP_SNOCVR_SNOMAD_YAML, self.task_config)
+        logger.debug(f"{self.task_config.PREP_SNOCVR_SNOMAD_YAML}:\n{pformat(prep_snocvr_snomad_config)}")
+
+        # define these locations in gdas/snow/prep/prep_snocvr_snomad.yaml.j2
+        logger.info("Copying SNOCVR and SNOMAD obs to DATA")
+        FileHandler(prep_snocvr_snomad_config.stage).sync()
+
+        # Execute obsBuilder to create the combined snocvr and snomad in IODA format
+        logger.info("Create the combined snocvr and snomad data in IODA format")
+
+        input_snocvr = f'{self.task_config.OPREFIX}snocvr.tm00.bufr_d'
+        input_snomad = f'{self.task_config.OPREFIX}snomad.tm00.bufr_d'
+        output_file = f'{self.task_config.OPREFIX}snocvr_snomad.tm00.nc'
+        if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"):
+            rm_p(output_file)
+
+        logger.info("Link OBSBUILDER into DATA/")
+        exe_src = self.task_config.OBSBUILDER
+        exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src))
+        if os.path.exists(exe_dest):
+            rm_p(exe_dest)
+        os.symlink(exe_src, exe_dest)
+
+        exe = Executable(exe_dest)
+        if os.path.exists(input_snocvr):
+            exe.add_default_arg(["--input_snocvr", f"{os.path.join(self.task_config.DATA, input_snocvr)}"])
+        exe.add_default_arg(["--output", f"{os.path.join(self.task_config.DATA, output_file)}"])
+        if os.path.exists(input_snomad):
+            exe.add_default_arg(["--input_snomad", f"{os.path.join(self.task_config.DATA, input_snomad)}"])
+        try:
+            logger.debug(f"Executing {exe}")
+            exe()
+        except OSError:
+            logger.exception(f"Failed to execute {exe}")
+            raise
+        except Exception as err:
+            logger.exception(f"An error occured during execution of {exe}")
+            raise WorkflowException(f"An error occured during execution of {exe}") from err
+
+        # Ensure the IODA snow depth SNOCVR+SNOMAD file is produced by the obsBuilder
+        # If so, copy to DATA/prep/
+        if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"):
+            logger.warning(f"{output_file} not produced - continuing without it.")
+        else:
+            logger.info(f"Copy {output_file} successfully generated")
+            FileHandler(prep_snocvr_snomad_config.netcdf).sync()
+
     @logit(logger)
     def add_increments(self) -> None:
         """Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds