From fa1cd40b6810eb9557d29a9487baa06d74ef5eab Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Thu, 11 Apr 2024 23:44:24 -0500
Subject: [PATCH 1/4] Add basic mean and spread for GEFS atmos products

Refs: #2296
---
 jobs/JGEFS_ATMOS_ENSSTAT              | 48 +++++++++++++
 jobs/rocoto/atmos_ensstat.sh          | 35 ++++++++++
 parm/config/gefs/config.atmos_ensstat | 11 +++
 parm/config/gefs/config.resources     | 29 ++++----
 scripts/exglobal_atmos_ensstat.sh     | 19 +++++
 sorc/link_workflow.sh                 |  3 +-
 ush/atmos_ensstat.sh                  | 99 +++++++++++++++++++++++++++
 workflow/applications/gefs.py         |  5 +-
 workflow/rocoto/gefs_tasks.py         | 39 +++++++++++
 9 files changed, 269 insertions(+), 19 deletions(-)
 create mode 100755 jobs/JGEFS_ATMOS_ENSSTAT
 create mode 100755 jobs/rocoto/atmos_ensstat.sh
 create mode 100644 parm/config/gefs/config.atmos_ensstat
 create mode 100755 scripts/exglobal_atmos_ensstat.sh
 create mode 100755 ush/atmos_ensstat.sh

diff --git a/jobs/JGEFS_ATMOS_ENSSTAT b/jobs/JGEFS_ATMOS_ENSSTAT
new file mode 100755
index 00000000000..eeef0ad04dd
--- /dev/null
+++ b/jobs/JGEFS_ATMOS_ENSSTAT
@@ -0,0 +1,48 @@
+#! /usr/bin/env bash
+
+#
+# Caculate the mean, spread, and other probabilistic fields.
+#
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "atmos_ensstat" -c "base atmos_ensstat"
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Construct COM variables from templates
+# Input directories loop over members, so this is done downstream
+
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COMOUT_ATMOS_GRIB_${grid}"
+  MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+done
+
+###############################################################
+# Run exglobal script
+"${SCRgfs}/exglobal_atmos_ensstat.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA:-NO}" = "NO" ]] && rm -rf "${DATA}"
+
+
+exit 0
diff --git a/jobs/rocoto/atmos_ensstat.sh b/jobs/rocoto/atmos_ensstat.sh
new file mode 100755
index 00000000000..284d1847765
--- /dev/null
+++ b/jobs/rocoto/atmos_ensstat.sh
@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+## atmosphere products driver script
+## FHRLST : forecast hour list to post-process (e.g. -f001, f000, f000_f001_f002, ...)
+###############################################################
+
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+if (( status != 0 )); then exit "${status}"; fi
+
+export job="atmos_ensstat"
+export jobid="${job}.$$"
+
+###############################################################
+# shellcheck disable=SC2153,SC2001
+IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
+
+#---------------------------------------------------------------
+# Execute the JJOB
+for fhr in "${fhrs[@]}"; do
+    # The analysis fhr is -001.  Performing math on negative, leading 0 integers is tricky.
+    # The negative needs to be in front of "10#", so do some regex magic to make it happen.
+    fhr="10#${fhr}"
+    fhr=${fhr//10\#-/-10\#}
+    export FORECAST_HOUR=$(( fhr ))
+    "${HOMEgfs}/jobs/JGEFS_ATMOS_ENSSTAT"
+    status=$?
+    if (( status != 0 )); then exit "${status}"; fi
+done
+
+exit 0
diff --git a/parm/config/gefs/config.atmos_ensstat b/parm/config/gefs/config.atmos_ensstat
new file mode 100644
index 00000000000..d371f758870
--- /dev/null
+++ b/parm/config/gefs/config.atmos_ensstat
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.atmos_ensstat ##########
+# atmosphere grib2 enstat specific
+
+echo "BEGIN: config.atmos_ensstat"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" atmos_ensstat
+
+echo "END: config.atmos_ensstat"
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
index 0be7e864a19..9bf62cf5143 100644
--- a/parm/config/gefs/config.resources
+++ b/parm/config/gefs/config.resources
@@ -7,23 +7,6 @@
 if (( $# != 1 )); then
 
     echo "Must specify an input task argument to set resource variables!"
-    echo "argument can be any one of the following:"
-    echo "stage_ic aerosol_init"
-    echo "prep prepsnowobs prepatmiodaobs"
-    echo "atmanlinit atmanlrun atmanlfinal"
-    echo "atmensanlinit atmensanlrun atmensanlfinal"
-    echo "snowanl"
-    echo "aeroanlinit aeroanlrun aeroanlfinal"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
-    echo "upp atmos_products"
-    echo "tracker genesis genesis_fsu"
-    echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
-    echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
-    echo "init_chem mom6ic"
-    echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
-    echo "wavegempak waveawipsbulls waveawipsgridded"
-    echo "postsnd awips gempak npoess"
-    echo "ocnanalprep prepoceanobs ocnanalbmat ocnanalrun ocnanalchkpt ocnanalpost ocnanalvrfy"
     exit 1
 
 fi
@@ -232,6 +215,18 @@ case ${step} in
     export is_exclusive=True
     ;;
 
+  "atmos_ensstat")
+    export wtime_atmos_ensstat="00:30:00"
+    export npe_atmos_ensstat=6
+    export nth_atmos_ensstat=1
+    export npe_node_atmos_ensstat="${npe_atmos_ensstat}"
+    export wtime_atmos_ensstat_gfs="${wtime_atmos_ensstat}"
+    export npe_atmos_ensstat_gfs="${npe_atmos_ensstat}"
+    export nth_atmos_ensstat_gfs="${nth_atmos_ensstat}"
+    export npe_node_atmos_ensstat_gfs="${npe_node_atmos_ensstat}"
+    export is_exclusive=True
+    ;;
+
   "oceanice_products")
     export wtime_oceanice_products="00:15:00"
     export npe_oceanice_products=1
diff --git a/scripts/exglobal_atmos_ensstat.sh b/scripts/exglobal_atmos_ensstat.sh
new file mode 100755
index 00000000000..7ad0ecfa30d
--- /dev/null
+++ b/scripts/exglobal_atmos_ensstat.sh
@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+fhr3=$(printf "%03d" "${FORECAST_HOUR}")
+
+if [[ -a mpmd_script ]]; then rm -Rf mpmd_script; fi
+
+{
+    for grid in '0p25' '0p50' '1p00'; do
+        echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3}"
+        # echo "${USHgfs}/atmos_ensstat.sh ${grid} ${fhr3} b"
+    done
+} > mpmd_script
+
+"${USHgfs}/run_mpmd.sh" mpmd_script
+err=$?
+
+exit "${err}"
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 988cd772615..45645aac9ba 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -274,7 +274,8 @@ cd "${HOMEgfs}/exec" || exit 1
 
 for utilexe in fbwndgfs.x gaussian_sfcanl.x gfs_bufr.x supvit.x syndat_getjtbul.x \
   syndat_maksynrc.x syndat_qctropcy.x tocsbufr.x overgridid.x rdbfmsua.x \
-  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x
+  mkgfsawps.x enkf_chgres_recenter_nc.x tave.x vint.x ocnicepost.x webtitle.x \
+  ensadd.x ensppf.x ensstat.x wave_stat.x
 do
   [[ -s "${utilexe}" ]] && rm -f "${utilexe}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/install/bin/${utilexe}" .
diff --git a/ush/atmos_ensstat.sh b/ush/atmos_ensstat.sh
new file mode 100755
index 00000000000..8cc8a7c8e38
--- /dev/null
+++ b/ush/atmos_ensstat.sh
@@ -0,0 +1,99 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+grid=${1}
+fhr3=${2}
+grid_type=${3:-''}
+
+mkdir "${grid}${grid_type}"
+cd "${grid}${grid_type}" || exit 2
+
+# Collect input grib files
+input_files=()
+for ((mem_num = 0; mem_num <= "${NMEM_ENS:-0}"; mem_num++)); do
+    mem=$(printf "%03d" "${mem_num}")
+    MEMDIR="mem${mem}" GRID="${grid}" YMD="${PDY}" HH="${cyc}" generate_com COMIN_ATMOS_GRIB:COM_ATMOS_GRIB_GRID_TMPL
+    memfile_in="${COMIN_ATMOS_GRIB}/${RUN}.t${cyc}z.pgrb2${grid_type}.${grid}.f${fhr3}"
+
+    if [[ -r "${memfile_in}.idx" ]]; then
+        ${NLN} "${memfile_in}" "mem${mem}"
+        input_files+=("mem${mem}")
+    else
+        echo "FATAL ERROR: ${memfile_in} does not exist"
+        exit 10
+    fi
+done
+
+num_found=${#input_files[@]}
+if (( num_found != NMEM_ENS + 1 )); then
+    echo "FATAL ERROR: Only ${num_found} grib files found out of $(( NMEM_ENS + 1 )) expected members."
+    exit 10
+fi
+
+# Create namelist for ensstat
+mean_out="${RUN}.t${cyc}z.pres_${grid_type}.ens_mean.${grid}.f${fhr3}.grib2"
+sd_out="${RUN}.t${cyc}z.pres_${grid_type}.ens_sd.${grid}.f${fhr3}.grib2"
+
+cat << EOF > input.nml
+&namdim
+    lfdim=${lfm:-''}
+/
+
+&namens
+    nfiles=${num_found}
+    nenspost=0
+    navg_min=${NMEM_ENS}
+
+    cfopg1="${mean_out}"
+    cfopg2="${sd_out}"
+
+$(
+    for (( filenum = 1; filenum <= num_found; filenum++ )); do
+        echo "    cfipg(${filenum})=\"${input_files[$((filenum-1))]}\","
+        echo "    iskip(${filenum})=0,"
+    done
+)
+/
+EOF
+
+cat input.nml
+
+# Run ensstat
+"${EXECgfs}/ensstat.x" < input.nml
+
+export err=$?
+if (( err != 0 )) ; then
+    echo "FATAL ERROR: ensstat returned error code ${err}"
+    exit "${err}"
+fi
+
+# Send data to com and send DBN alerts
+comout_var_name="COMOUT_ATMOS_GRIB_${grid}"
+comout_path="${!comout_var_name}"
+
+for outfile in ${mean_out} ${sd_out}; do
+    if [[ ! -s ${outfile} ]]; then
+        echo "FATAL ERROR: Failed to create ${outfile}"
+        exit 20
+    fi
+
+    ${WGRIB2} -s "${outfile}" > "${outfile}.idx"
+    err=$?
+    if (( err != 0 )); then
+        echo "FATAL ERROR: Failed to create inventory file, wgrib2 returned ${err}"
+        exit "${err}"
+    fi
+
+    cpfs "${outfile}" "${comout_path}/${outfile}"
+    cpfs "${outfile}.idx" "${comout_path}/${outfile}.idx"
+
+    if [[ ${SENDDBN} == "YES" ]]; then
+        "${DBNROOT}/bin/dbn_alert" MODEL "${RUN^^}_PGB2${grid_type}_${grid}" "${job}" \
+            "${comout_path}/${outfile}"
+        "${DBNROOT}/bin/dbn_alert" MODEL "${RUN^^}_PGB2${grid_type}_${grid}" "${job}" \
+            "${comout_path}/${outfile}.idx"
+    fi
+
+done
+
diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py
index 0be4dc71242..b14c1a90036 100644
--- a/workflow/applications/gefs.py
+++ b/workflow/applications/gefs.py
@@ -17,7 +17,7 @@ def _get_app_configs(self):
         configs = ['stage_ic', 'fcst', 'atmos_products']
 
         if self.nens > 0:
-            configs += ['efcs']
+            configs += ['efcs', 'atmos_ensstat']
 
         if self.do_wave:
             configs += ['waveinit', 'wavepostsbs', 'wavepostpnt']
@@ -52,6 +52,9 @@ def get_task_names(self):
 
         tasks += ['atmos_prod']
 
+        if self.nens > 0:
+            tasks += ['atmos_ensstat']
+
         if self.do_ocean:
             tasks += ['ocean_prod']
 
diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py
index 50b24f35780..5d1eda817b1 100644
--- a/workflow/rocoto/gefs_tasks.py
+++ b/workflow/rocoto/gefs_tasks.py
@@ -233,6 +233,45 @@ def _atmosoceaniceprod(self, component: str):
 
         return task
 
+    def atmos_ensstat(self):
+
+        resources = self.get_resource('atmos_ensstat')
+
+        deps = []
+        for member in range(0, self.nmem + 1):
+            task = f'atmos_prod_mem{member:03d}_f#fhr#'
+            dep_dict = {'type': 'task', 'name': task}
+            deps.append(rocoto.add_dependency(dep_dict))
+
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+
+        postenvars = self.envars.copy()
+        postenvar_dict = {'FHRLST': '#fhr#'}
+        for key, value in postenvar_dict.items():
+            postenvars.append(rocoto.create_envar(name=key, value=str(value)))
+
+        task_name = f'atmos_ensstat_f#fhr#'
+        task_dict = {'task_name': task_name,
+                     'resources': resources,
+                     'dependency': dependencies,
+                     'envars': postenvars,
+                     'cycledef': 'gefs',
+                     'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh',
+                     'job_name': f'{self.pslot}_{task_name}_@H',
+                     'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
+                     'maxtries': '&MAXTRIES;'}
+
+        fhrs = self._get_forecast_hours('gefs', self._configs['atmos_ensstat'])
+        fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])}
+
+        fhr_metatask_dict = {'task_name': f'atmos_ensstat',
+                             'task_dict': task_dict,
+                             'var_dict': fhr_var_dict}
+
+        task = rocoto.create_task(fhr_metatask_dict)
+
+        return task
+
     def wavepostsbs(self):
         deps = []
         for wave_grid in self._configs['wavepostsbs']['waveGRD'].split():

From 818ffc140e5bffd3c1cc6a9986604d2dd2d688b9 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Fri, 12 Apr 2024 11:34:47 -0500
Subject: [PATCH 2/4] Adjust filenames for EE2 compliance

---
 ush/atmos_ensstat.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ush/atmos_ensstat.sh b/ush/atmos_ensstat.sh
index 8cc8a7c8e38..1f13ab01f70 100755
--- a/ush/atmos_ensstat.sh
+++ b/ush/atmos_ensstat.sh
@@ -32,8 +32,8 @@ if (( num_found != NMEM_ENS + 1 )); then
 fi
 
 # Create namelist for ensstat
-mean_out="${RUN}.t${cyc}z.pres_${grid_type}.ens_mean.${grid}.f${fhr3}.grib2"
-sd_out="${RUN}.t${cyc}z.pres_${grid_type}.ens_sd.${grid}.f${fhr3}.grib2"
+mean_out="${RUN}.t${cyc}z.mean.pres_${grid_type}.${grid}.f${fhr3}.grib2"
+spr_out="${RUN}.t${cyc}z.spread.pres_${grid_type}.${grid}.f${fhr3}.grib2"
 
 cat << EOF > input.nml
 &namdim
@@ -46,7 +46,7 @@ cat << EOF > input.nml
     navg_min=${NMEM_ENS}
 
     cfopg1="${mean_out}"
-    cfopg2="${sd_out}"
+    cfopg2="${spr_out}"
 
 $(
     for (( filenum = 1; filenum <= num_found; filenum++ )); do
@@ -72,7 +72,7 @@ fi
 comout_var_name="COMOUT_ATMOS_GRIB_${grid}"
 comout_path="${!comout_var_name}"
 
-for outfile in ${mean_out} ${sd_out}; do
+for outfile in ${mean_out} ${spr_out}; do
     if [[ ! -s ${outfile} ]]; then
         echo "FATAL ERROR: Failed to create ${outfile}"
         exit 20

From d138fc652381d321f13983ee3ac2577423fd5ccf Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Fri, 12 Apr 2024 11:46:02 -0500
Subject: [PATCH 3/4] Switch j-job name back to JGLOBAL

---
 jobs/{JGEFS_ATMOS_ENSSTAT => JGLOBAL_ATMOS_ENSSTAT} | 0
 jobs/rocoto/atmos_ensstat.sh                        | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename jobs/{JGEFS_ATMOS_ENSSTAT => JGLOBAL_ATMOS_ENSSTAT} (100%)

diff --git a/jobs/JGEFS_ATMOS_ENSSTAT b/jobs/JGLOBAL_ATMOS_ENSSTAT
similarity index 100%
rename from jobs/JGEFS_ATMOS_ENSSTAT
rename to jobs/JGLOBAL_ATMOS_ENSSTAT
diff --git a/jobs/rocoto/atmos_ensstat.sh b/jobs/rocoto/atmos_ensstat.sh
index 284d1847765..cbaaff9a0f9 100755
--- a/jobs/rocoto/atmos_ensstat.sh
+++ b/jobs/rocoto/atmos_ensstat.sh
@@ -27,7 +27,7 @@ for fhr in "${fhrs[@]}"; do
     fhr="10#${fhr}"
     fhr=${fhr//10\#-/-10\#}
     export FORECAST_HOUR=$(( fhr ))
-    "${HOMEgfs}/jobs/JGEFS_ATMOS_ENSSTAT"
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_ENSSTAT"
     status=$?
     if (( status != 0 )); then exit "${status}"; fi
 done

From 0393b2bb55a3287b615d90418de8a25b5c647c1e Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 15 Apr 2024 23:01:46 -0500
Subject: [PATCH 4/4] Change generate_com to declare_from_tmpl in ensstat

---
 jobs/JGLOBAL_ATMOS_ENSSTAT | 2 +-
 ush/atmos_ensstat.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/jobs/JGLOBAL_ATMOS_ENSSTAT b/jobs/JGLOBAL_ATMOS_ENSSTAT
index eeef0ad04dd..e09410d5812 100755
--- a/jobs/JGLOBAL_ATMOS_ENSSTAT
+++ b/jobs/JGLOBAL_ATMOS_ENSSTAT
@@ -17,7 +17,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmos_ensstat" -c "base atmos_ensstat
 
 for grid in '0p25' '0p50' '1p00'; do
   prod_dir="COMOUT_ATMOS_GRIB_${grid}"
-  MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  MEMDIR="ensstat" GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
   if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
 done
 
diff --git a/ush/atmos_ensstat.sh b/ush/atmos_ensstat.sh
index 1f13ab01f70..17981a8c3e8 100755
--- a/ush/atmos_ensstat.sh
+++ b/ush/atmos_ensstat.sh
@@ -13,7 +13,7 @@ cd "${grid}${grid_type}" || exit 2
 input_files=()
 for ((mem_num = 0; mem_num <= "${NMEM_ENS:-0}"; mem_num++)); do
     mem=$(printf "%03d" "${mem_num}")
-    MEMDIR="mem${mem}" GRID="${grid}" YMD="${PDY}" HH="${cyc}" generate_com COMIN_ATMOS_GRIB:COM_ATMOS_GRIB_GRID_TMPL
+    MEMDIR="mem${mem}" GRID="${grid}" YMD="${PDY}" HH="${cyc}" declare_from_tmpl COMIN_ATMOS_GRIB:COM_ATMOS_GRIB_GRID_TMPL
     memfile_in="${COMIN_ATMOS_GRIB}/${RUN}.t${cyc}z.pgrb2${grid_type}.${grid}.f${fhr3}"
 
     if [[ -r "${memfile_in}.idx" ]]; then