JEDI-based ensemble recentering and analysis calculation

env/HERA.env

@@ -157,6 +157,17 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF=${APRUN_default}
 
+elif [[ "${step}" = "ecen_fv3jedi" ]]; then
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/HERCULES.env

@@ -161,6 +161,17 @@ case ${step} in
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
  ;;
+ "ecen_fv3jedi")
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+ ;;
+ "analcalc_fv3jedi")
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+ ;;
  "anal" | "analcalc")
 
     export MKL_NUM_THREADS=4

env/JET.env

@@ -130,6 +130,17 @@ elif [[ "${step}" = "marineanlvar" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
     export APRUN_MARINEANLVAR="${APRUN_default}"
 
+elif [[ "${step}" = "ecen_fv3jedi" ]]; then
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/ORION.env

@@ -158,6 +158,17 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
 
+elif [[ "${step}" = "ecen_fv3jedi" ]]; then
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/S4.env

@@ -126,6 +126,17 @@ elif [[ "${step}" = "marinebmat" ]]; then
 elif [[ "${step}" = "marinerun" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 
+elif [[ "${step}" = "ecen_fv3jedi" ]]; then
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpus-per-task=${NTHREADS_ECEN_FV3JEDI}"
+
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

env/WCOSS2.env

@@ -88,7 +88,7 @@ elif [[ "${step}" = "aeroanlgenb" ]]; then
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}
-    export APRUN_PREPOBSAERO="${APRUN_default} --ppn ${tasks_per_node}--cpu-bind depth --depth=${NTHREADS_PREPOBSAERO}"
+    export APRUN_PREPOBSAERO="${APRUN_default} --ppn ${tasks_per_node} --cpu-bind depth --depth ${NTHREADS_PREPOBSAERO}"
 
 elif [[ "${step}" = "snowanl" ]]; then
 
@@ -139,6 +139,17 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default}"
 
+elif [[ "${step}" = "ecen_fv3jedi" ]]; then
+
+    export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}
+    export APRUN_CORRECTION_INCREMENT="${launcher} -n ${ntasks_correction_increment} --cpu-bind depth --depth ${NTHREADS_ECEN_FV3JEDI}"
+    export APRUN_ENSEMBLE_RECENTER="${launcher} -n ${ntasks_ensemble_recenter} --cpu-bind depth --depth ${NTHREADS_ECEN_FV3JEDI}"
+
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export OMP_PLACES=cores

jobs/JGDAS_ENKF_ECEN_FV3JEDI

@@ -0,0 +1,67 @@
+#! /usr/bin/env bash
+
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "ecen_fv3jedi" -c "base ecen_fv3jedi"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+GDUMP="gdas"
+GDUMP_ENS="enkfgdas"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} RUN=${GDUMP} declare_from_tmpl -rx \
+   COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+YMD=${PDY} HH=${cyc} RUN=${GDUMP} declare_from_tmpl -rx \
+   COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL
+MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMOUT_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+   COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+
+##############################################
+# Run relevant script
+##############################################
+
+EXSCRIPT=${GDASATMRUNPY:-${SCRgfs}/exgdas_enkf_ecen_fv3jedi.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+  exit "${status}"
+fi
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##############################################
+# Remove the Temporary working directory
+##############################################
+
+cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 )
+if [[ ${KEEPDATA} = "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI

@@ -0,0 +1,67 @@
+#! /usr/bin/env bash
+
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+# shellcheck disable=SC2312
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc_fv3jedi" -c "base analcalc_fv3jedi"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+GDUMP="gdas"
+export rCDUMP="${RUN/enkf}"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} RUN=${RUN} declare_from_tmpl -rx \
+   COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+YMD=${PDY} HH=${cyc} RUN=${RUN} declare_from_tmpl -rx \
+   COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+   COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+
+##############################################
+# Run relevant script
+##############################################
+
+EXSCRIPT=${GDASATMRUNPY:-${SCRgfs}/exglobal_atmos_analysis_calc_fv3jedi.py}
+${EXSCRIPT}
+status=$?
+if [[ ${status} -ne 0 ]]; then
+  exit "${status}"
+fi
+
+# Write analysis log file
+echo "${rCDUMP} ${PDY}${cyc} atmanl and sfcanl done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.loganl.txt"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##############################################
+# Remove the Temporary working directory
+##############################################
+
+cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 )
+if [[ ${KEEPDATA} = "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/rocoto/analcalc_fv3jedi.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="analcalc_fv3jedi"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI
+status=$?
+exit "${status}"

jobs/rocoto/ecen_fv3jedi.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="ecen_fv3jedi"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGDAS_ENKF_ECEN_FV3JEDI
+status=$?
+exit "${status}"

parm/archive/enkf.yaml.j2

@@ -10,22 +10,26 @@ enkf:
         {% for fhr in range(fhmin, fhmax + 1, fhout) %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}_epos{{ '%03d' % (fhr - fhmin) }}.log"
         {% endfor %}
+        {% if not DO_JEDIATMENS %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}_echgres.log"
         {% endif %}
+        {% endif %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}_esfc.log"
+        {% if not DO_JEDIATMENS %}
         {% for grp in range(IAUFHRS | length) %}
         - "logs/{{ cycle_YMDH }}/{{ RUN }}_ecen{{ '%03d' % grp }}.log"
         {% endfor %}
+        {% endif %}
 
         {% if lobsdiag_forenkf %}
            {% if DO_JEDIATMENS %}
-              {% set steps = ["atmensanlinit", "atmensanlobs", "atmensanlsol", "atmensanlfv3inc", "atmensanlfinal"] %}
+              {% set steps = ["atmensanlinit", "atmensanlobs", "atmensanlsol", "atmensanlfv3inc", "atmensanlfinal", "ecen_fv3jedi"] %}
            {% else %}
               {% set steps = ["eobs", "ediag", "eupd"] %}
            {% endif %}
         {% else %}
            {% if DO_JEDIATMENS %}
-              {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %}
+              {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal", "ecen_fv3jedi"] %}
            {% else %}
               {% set steps = ["eobs", "eupd"] %}
               {% for mem in range(1, nmem_ens + 1) %}
@@ -61,12 +65,14 @@ enkf:
         {% set da_files = ["atmensanlobs.yaml",
                            "atmensanlsol.yaml",
                            "atmensanlfv3inc.yaml",
-                           "atminc.ensmean.nc",
+                           "correction_increment.yaml",
+                           "ensemble_recenter.yaml",
                            "atmensstat"] %}
         {% else %}
         {% set da_files = ["atmensanlletkf.yaml",
                            "atmensanlfv3inc.yaml",
-                           "atminc.ensmean.nc",
+                           "correction_increment.yaml",
+                           "ensemble_recenter.yaml",
                            "atmensstat"] %}
         {% endif %}
         {% endif %}
@@ -84,23 +90,33 @@ enkf:
 
         # Ensemble mean analyses/increments
         # 6-hr analysis/increment
+        {% if not DO_JEDIATMENS %}
         {% if do_calc_increment %}
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmanl.ensmean.nc"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atminc.ensmean.nc"
         {% endif %}
+        {% else %}
+        - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}cubed_sphere_grid_atmanl.ensmean.nc"
+        {% endif %}
 
         {% if DOIAU %}
         # IAU increments/analyses
 
         {% for fhr in IAUFHRS if fhr != 6 %}
+        {% if not DO_JEDIATMENS %}
         {% if do_calc_increment %}
         # Store analyses instead of increments
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atma{{ '%03d' % fhr }}.ensmean.nc"
         {% else %}
+        {% if not DO_JEDIATMENS %}
         # Store increments
         - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}atmi{{ '%03d' % fhr }}.ensmean.nc"
         {% endif %}
+        {% endif %}
+        {% else %}
+        - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}cubed_sphere_grid_atma{{ '%03d' % fhr }}.ensmean.nc"
+        {% endif %}
         {% endfor %}
 
         {% endif %}  # End IAU

parm/archive/enkf_grp.yaml.j2

@@ -23,9 +23,15 @@ enkf_grp:
         {% if not lobsdiag_forenkf %}
         - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat"
         {% endif %}
+        {% if DO_JEDIATMENS %}
+        {% for itile in range(6) %}
+        - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}cubed_sphere_grid_ratminc.tile{{ itile+1 }}.nc"
+        {% endfor %}
+        {% else %}
         {% if do_calc_increment %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}atmanl.nc"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}ratminc.nc"
         {% endif %}
+        {% endif %}
         {% endfor %}  # first_group_mem to last_group_mem