From 174561901af59d0fa5e41b8186b43045f24c1e22 Mon Sep 17 00:00:00 2001
From: Jiarui Dong <Jiarui.Dong@noaa.gov>
Date: Mon, 18 Mar 2024 18:01:52 -0400
Subject: [PATCH 1/4] Add the initial configure to prepare obs for the ensemble
 members.

---
 env/HERA.env                           |  6 ++--
 jobs/JGLOBAL_PREP_SNOW_OBS             | 30 ++++++++++++++++----
 parm/config/gfs/config.com             |  2 +-
 parm/config/gfs/config.epsn            | 13 +++++++++
 parm/config/gfs/config.resources       | 12 ++++----
 ush/python/pygfs/task/snow_analysis.py | 17 +++++------
 workflow/applications/gfs_cycled.py    |  4 +++
 workflow/rocoto/gfs_tasks.py           | 39 ++++++++++++++++++++++++++
 8 files changed, 100 insertions(+), 23 deletions(-)
 create mode 100644 parm/config/gfs/config.epsn

diff --git a/env/HERA.env b/env/HERA.env
index f55434e8d9d..9d9411ddf53 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -4,9 +4,9 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanlrun atmensanlrun aeroanlrun snowanl"
+    echo "atmanlrun atmensanlrun aeroanlrun prepsnowobs snowanl"
     echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
+    echo "eobs eupd ecen efcs epos epsn"
     echo "postsnd awips gempak"
     exit 1
 
@@ -42,7 +42,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="HERA"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "prepsnowobs" ]]; then
+elif [[ "${step}" = "prepsnowobs" ]] || [[ "${step}" = "epsn" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS
index 1fc7e3e5c33..920557c8463 100755
--- a/jobs/JGLOBAL_PREP_SNOW_OBS
+++ b/jobs/JGLOBAL_PREP_SNOW_OBS
@@ -1,8 +1,20 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-export DATA=${DATA:-${DATAROOT}/${RUN}snowanl_${cyc}}
-source "${HOMEgfs}/ush/jjob_header.sh" -e "prepsnowobs" -c "base prepsnowobs"
+if (( 10#${ENSMEM:-0} > 0 )); then
+  source "${HOMEgfs}/ush/jjob_header.sh" -e "epsn" -c "base prepsnowobs epsn"
+  export CASE=${CASE_ENS}
+else
+  source "${HOMEgfs}/ush/jjob_header.sh" -e "prepsnowobs" -c "base prepsnowobs"
+fi
+
+# Restart conditions for GFS cycle come from GDAS
+rCDUMP=${RUN}
+[[ ${RUN} == "gfs" ]] && export rCDUMP="gdas"
+
+# Get OBS for ENKFGDAS run from GDAS
+oCDUMP=${RUN}
+[[ ${RUN} == "enkfgdas" ]] && export oCDUMP="gdas"
 
 ##############################################
 # Set variables used in the script
@@ -12,17 +24,19 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "prepsnowobs" -c "base prepsnowobs"
 GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
 gPDY=${GDATE:0:8}
 gcyc=${GDATE:8:2}
-GDUMP="gdas"
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS
+RUN=${oCDUMP} MEMDIR="" YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS_IN:COM_OBS_TMPL
+RUN=${RUN} YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS_OUT:COM_OBS_TMPL
 
-RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
     COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
+mkdir -m 775 -p "${COM_OBS_OUT}"
+
 ###############################################################
 # Run relevant script
 EXSCRIPT=${GDASSNOWPREPPY:-${SCRgfs}/exglobal_prep_snow_obs.py}
@@ -41,4 +55,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || true
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0
diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com
index 2f99e709ea4..f6a1790c906 100644
--- a/parm/config/gfs/config.com
+++ b/parm/config/gfs/config.com
@@ -38,7 +38,7 @@ if [[ "${RUN_ENVIR:-emc}" == "nco" ]]; then
     COM_OBS_TMPL=$(compath.py "${envir}/obsproc/${obsproc_ver}")'/${RUN}.${YMD}/${HH}/atmos'
     COM_RTOFS_TMPL=$(compath.py "${envir}/${WAVECUR_DID}/${rtofs_ver}")
 else
-    COM_OBS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}/obs'
+    COM_OBS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}/obs/${MEMDIR}'
     COM_RTOFS_TMPL='${DMPDIR}'
 fi
 declare -rx COM_OBS_TMPL COM_RTOFS_TMPL
diff --git a/parm/config/gfs/config.epsn b/parm/config/gfs/config.epsn
new file mode 100644
index 00000000000..f638cc04de6
--- /dev/null
+++ b/parm/config/gfs/config.epsn
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+########## config.epsn ##########
+# Snow Obs Prep specific
+
+echo "BEGIN: config.epsn"
+
+source "${EXPDIR}/config.prepsnowobs"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" epsn
+
+echo "END: config.epsn"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 21f0a111ef3..47572a9b1f0 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -18,7 +18,7 @@ if (( $# != 1 )); then
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
-    echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
+    echo "eobs ediag eomg eupd ecen esfc efcs epos earc epsn"
     echo "init_chem mom6ic oceanice_products"
     echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
     echo "wavegempak waveawipsbulls waveawipsgridded"
@@ -83,11 +83,11 @@ case ${step} in
     fi
     ;;
 
-  "prepsnowobs")
-    export wtime_prepsnowobs="00:05:00"
-    export npe_prepsnowobs=1
-    export nth_prepsnowobs=1
-    export npe_node_prepsnowobs=1
+  "prepsnowobs" | "epsn")
+    declare -x "wtime_${step}"="00:05:00"
+    declare -x "npe_${step}"=1
+    declare -x "nth_${step}"=1
+    declare -x "npe_node_${step}"=1
     ;;
 
   "prepatmiodaobs")
diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py
index c149f140b61..cdd0ff6e517 100644
--- a/ush/python/pygfs/task/snow_analysis.py
+++ b/ush/python/pygfs/task/snow_analysis.py
@@ -45,7 +45,8 @@ def __init__(self, config):
                 'npz': self.config.LEVS - 1,
                 'SNOW_WINDOW_BEGIN': _window_begin,
                 'SNOW_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H",
-                'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.",
+                'OPREFIX_IN': f"{self.runtime_config.RUN.replace('enkf','')}.t{self.runtime_config.cyc:02d}z.",
+                'OPREFIX_OUT': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.",
                 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.",
                 'jedi_yaml': _letkfoi_yaml
             }
@@ -73,8 +74,8 @@ def prepare_GTS(self) -> None:
 
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
-        keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'OCNRES', 'ntiles']
+        keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS_IN', 'COM_OBS_OUT', 'COM_ATMOS_RESTART_PREV',
+                'OPREFIX_IN', 'OPREFIX_OUT', 'CASE', 'OCNRES', 'ntiles']
         for key in keys:
             localconf[key] = self.task_config[key]
 
@@ -126,7 +127,7 @@ def _gtsbufr2iodax(exe, yaml_file):
         # Ensure the IODA snow depth GTS file is produced by the IODA converter
         # If so, copy to COM_OBS/
         try:
-            FileHandler(prep_gts_config.gtsioda).sync()
+            FileHandler(prep_gts_config.gtsioda[self.runtime_config.RUN]).sync()
         except OSError as err:
             logger.exception(f"{self.task_config.BUFR2IODAX} failed to produce GTS ioda files")
             raise OSError(err)
@@ -152,8 +153,8 @@ def prepare_IMS(self) -> None:
 
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
-        keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'OCNRES', 'ntiles', 'FIXgfs']
+        keys = ['DATA', 'current_cycle', 'COM_OBS_IN', 'COM_OBS_OUT', 'COM_ATMOS_RESTART_PREV',
+                'OPREFIX_IN', 'OPREFIX_OUT', 'CASE', 'OCNRES', 'ntiles', 'FIXgfs']
         for key in keys:
             localconf[key] = self.task_config[key]
 
@@ -227,7 +228,7 @@ def prepare_IMS(self) -> None:
             logger.exception(f"{self.task_config.IMS2IODACONV} failed to produce {output_file}")
             raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}")
         else:
-            logger.info(f"Copy {output_file} to {self.task_config.COM_OBS}")
+            logger.info(f"Copy {output_file} to {self.task_config.COM_OBS_OUT}")
             FileHandler(prep_ims_config.ims2ioda).sync()
 
     @logit(logger)
@@ -249,7 +250,7 @@ def initialize(self) -> None:
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
         keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'OCNRES', 'ntiles']
+                'OPREFIX_OUT', 'CASE', 'OCNRES', 'ntiles']
         for key in keys:
             localconf[key] = self.task_config[key]
 
diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py
index 51b430a996a..130692fee4b 100644
--- a/workflow/applications/gfs_cycled.py
+++ b/workflow/applications/gfs_cycled.py
@@ -58,6 +58,8 @@ def _get_app_configs(self):
                 configs += ['atmensanlinit', 'atmensanlrun', 'atmensanlfinal']
             else:
                 configs += ['eobs', 'eomg', 'ediag', 'eupd']
+            if self.do_jedisnowda:
+                configs += ['epsn']
             configs += ['ecen', 'esfc', 'efcs', 'echgres', 'epos', 'earc']
 
         if self.do_fit2obs:
@@ -163,6 +165,8 @@ def get_task_names(self):
             else:
                 hybrid_tasks += ['eobs', 'eupd', 'echgres']
                 hybrid_tasks += ['ediag'] if self.lobsdiag_forenkf else ['eomg']
+            if self.do_jedisnowda:
+                hybrid_tasks += ['epsn']
             hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup']
 
         # Collect all "gdas" cycle tasks
diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index 36b57bd6dc4..7ea2c1ca529 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -2413,6 +2413,45 @@ def atmensanlfinal(self):
 
         return task
 
+    def epsn(self):
+
+        deps = []
+        dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}prep'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        epsnenvars = self.envars.copy()
+        epsnenvars_dict = {'ENSMEM': '#member#',
+                           'MEMDIR': 'mem#member#'
+                           }
+        for key, value in epsnenvars_dict.items():
+            epsnenvars.append(rocoto.create_envar(name=key, value=str(value)))
+
+        resources = self.get_resource('epsn')
+        task_name = f'{self.cdump}prepsnowobs_mem#member#'
+        task_dict = {'task_name': task_name,
+                     'resources': resources,
+                     'dependency': dependencies,
+                     'envars': epsnenvars,
+                     'cycledef': self.cdump.replace('enkf', ''),
+                     'command': f'{self.HOMEgfs}/jobs/rocoto/prepsnowobs.sh',
+                     'job_name': f'{self.pslot}_{task_name}_@H',
+                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
+                     'maxtries': '&MAXTRIES;'
+                     }
+
+        member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(1, self.nmem + 1)])}
+        metatask_dict = {'task_name': f'{self.cdump}prepsnowobs',
+                         'var_dict': member_var_dict,
+                         'task_dict': task_dict
+                         }
+
+        task = rocoto.create_task(metatask_dict)
+
+        return task
+
     def ecen(self):
 
         def _get_ecengroups():

From 8ef510484cedc613cc32eb3c533f9dc1b41e621e Mon Sep 17 00:00:00 2001
From: Jiarui Dong <Jiarui.Dong@noaa.gov>
Date: Sat, 6 Apr 2024 11:24:05 -0400
Subject: [PATCH 2/4] Change COM_OBS_IN to COMIN_OBS and change COM_OBS_OUT to
 COMOUT_OBS.

---
 jobs/JGLOBAL_PREP_SNOW_OBS             | 6 +++---
 ush/python/pygfs/task/snow_analysis.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS
index 920557c8463..55de239e8cf 100755
--- a/jobs/JGLOBAL_PREP_SNOW_OBS
+++ b/jobs/JGLOBAL_PREP_SNOW_OBS
@@ -29,13 +29,13 @@ gcyc=${GDATE:8:2}
 # Begin JOB SPECIFIC work
 ##############################################
 # Generate COM variables from templates
-RUN=${oCDUMP} MEMDIR="" YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS_IN:COM_OBS_TMPL
-RUN=${RUN} YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS_OUT:COM_OBS_TMPL
+RUN=${oCDUMP} MEMDIR="" YMD=${PDY} HH=${cyc} generate_com -rx COMIN_OBS:COM_OBS_TMPL
+RUN=${RUN} YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL
 
 RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
     COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
-mkdir -m 775 -p "${COM_OBS_OUT}"
+mkdir -m 775 -p "${COMOUT_OBS}"
 
 ###############################################################
 # Run relevant script
diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py
index cdd0ff6e517..3020019388f 100644
--- a/ush/python/pygfs/task/snow_analysis.py
+++ b/ush/python/pygfs/task/snow_analysis.py
@@ -74,7 +74,7 @@ def prepare_GTS(self) -> None:
 
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
-        keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS_IN', 'COM_OBS_OUT', 'COM_ATMOS_RESTART_PREV',
+        keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COMIN_OBS', 'COMOUT_OBS', 'COM_ATMOS_RESTART_PREV',
                 'OPREFIX_IN', 'OPREFIX_OUT', 'CASE', 'OCNRES', 'ntiles']
         for key in keys:
             localconf[key] = self.task_config[key]
@@ -153,7 +153,7 @@ def prepare_IMS(self) -> None:
 
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
-        keys = ['DATA', 'current_cycle', 'COM_OBS_IN', 'COM_OBS_OUT', 'COM_ATMOS_RESTART_PREV',
+        keys = ['DATA', 'current_cycle', 'COMIN_OBS', 'COMOUT_OBS', 'COM_ATMOS_RESTART_PREV',
                 'OPREFIX_IN', 'OPREFIX_OUT', 'CASE', 'OCNRES', 'ntiles', 'FIXgfs']
         for key in keys:
             localconf[key] = self.task_config[key]
@@ -228,7 +228,7 @@ def prepare_IMS(self) -> None:
             logger.exception(f"{self.task_config.IMS2IODACONV} failed to produce {output_file}")
             raise FileNotFoundError(f"{os.path.join(localconf.DATA, output_file)}")
         else:
-            logger.info(f"Copy {output_file} to {self.task_config.COM_OBS_OUT}")
+            logger.info(f"Copy {output_file} to {self.task_config.COMOUT_OBS}")
             FileHandler(prep_ims_config.ims2ioda).sync()
 
     @logit(logger)

From 15bbd454b9156bc8b172966669277cce83b76eb0 Mon Sep 17 00:00:00 2001
From: Jiarui Dong <Jiarui.Dong@noaa.gov>
Date: Wed, 10 Apr 2024 18:07:31 -0400
Subject: [PATCH 3/4] Change generate_com to declare_from_tmpl.

---
 jobs/JGLOBAL_PREP_SNOW_OBS | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS
index 55de239e8cf..caa4126ebfe 100755
--- a/jobs/JGLOBAL_PREP_SNOW_OBS
+++ b/jobs/JGLOBAL_PREP_SNOW_OBS
@@ -8,39 +8,46 @@ else
   source "${HOMEgfs}/ush/jjob_header.sh" -e "prepsnowobs" -c "base prepsnowobs"
 fi
 
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
 # Restart conditions for GFS cycle come from GDAS
 rCDUMP=${RUN}
-[[ ${RUN} == "gfs" ]] && export rCDUMP="gdas"
+export rCDUMP="${RUN/gfs/gdas}"
 
 # Get OBS for ENKFGDAS run from GDAS
 oCDUMP=${RUN}
-[[ ${RUN} == "enkfgdas" ]] && export oCDUMP="gdas"
+export oCDUMP="${RUN/enkfgdas/gdas}"
 
 ##############################################
 # Set variables used in the script
 ##############################################
 # Ignore possible spelling error (nothing is misspelled)
 # shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-gPDY=${GDATE:0:8}
-gcyc=${GDATE:8:2}
+GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H)
+declare -rx GDATE
+# shellcheck disable=
+declare -rx gPDY="${GDATE:0:8}"
+declare -rx gcyc="${GDATE:8:2}"
+
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 # Generate COM variables from templates
-RUN=${oCDUMP} MEMDIR="" YMD=${PDY} HH=${cyc} generate_com -rx COMIN_OBS:COM_OBS_TMPL
-RUN=${RUN} YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL
+RUN=${oCDUMP} MEMDIR="" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL
+RUN=${RUN} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMOUT_OBS:COM_OBS_TMPL
 
-RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \
+RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
 
 mkdir -m 775 -p "${COMOUT_OBS}"
 
 ###############################################################
 # Run relevant script
-EXSCRIPT=${GDASSNOWPREPPY:-${SCRgfs}/exglobal_prep_snow_obs.py}
-${EXSCRIPT}
+###############################################################
+${GDASSNOWPREPPY:-${SCRgfs}/exglobal_prep_snow_obs.py}
 status=$?
 [[ ${status} -ne 0 ]] && (echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}")
 

From 2557cb7357031749a26ced101f2320c93e254a6a Mon Sep 17 00:00:00 2001
From: Jiarui Dong <Jiarui.Dong@noaa.gov>
Date: Tue, 16 Apr 2024 17:33:49 -0400
Subject: [PATCH 4/4] Update the GDASApp commit hash.

---
 sorc/gdas.cd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/gdas.cd b/sorc/gdas.cd
index 2198b419567..f5d6da3844b 160000
--- a/sorc/gdas.cd
+++ b/sorc/gdas.cd
@@ -1 +1 @@
-Subproject commit 2198b419567cf7efa7404cd076e76e01d86f9e58
+Subproject commit f5d6da3844b63847f1c89f2b845f8b330302b936