Add the mean EnKF soil increment to the deterministic member

.github/CODEOWNERS

@@ -181,6 +181,7 @@ syndat_qctropcy.sh @JiayiPeng-NOAA
 tropcy_relocate.sh @JiayiPeng-NOAA 
 tropcy_relocate_extrkr.sh @JiayiPeng-NOAA 
 wave_*.sh @JessicaMeixner-NOAA @sbanihash
+regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA
 
 # ush/python
 ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA

env/GAEAC5.env

@@ -175,6 +175,14 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

env/GAEAC6.env

@@ -175,6 +175,14 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

env/HERA.env

@@ -183,6 +183,13 @@ elif [[ "${step}" = "sfcanl" ]]; then
     export NTHREADS_CYCLE=${threads_per_task:-14}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -248,6 +255,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/HERCULES.env

@@ -186,6 +186,13 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
  ;;
  "eobs")
 
@@ -262,6 +269,13 @@ case ${step} in
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "epos")
 

env/JET.env

@@ -156,6 +156,13 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -221,6 +228,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/ORION.env

@@ -184,6 +184,13 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -250,6 +257,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/WCOSS2.env

@@ -175,6 +175,13 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export OMP_PLACES=cores
@@ -260,6 +267,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

jobs/JGLOBAL_ATMOS_SFCANL

@@ -24,6 +24,9 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
+RUN="enkfgdas" MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+  COMIN_ATMOS_ENKF_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL
+
 mkdir -p "${COMOUT_ATMOS_RESTART}"
 
 

jobs/rocoto/esfc.sh

@@ -4,7 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# CSD for GSI_SOILANAL, need ufsda modules (temporary, until merged)
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

jobs/rocoto/sfcanl.sh

@@ -4,7 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# CSD for GSI_SOILANAL, need ufsda modules (temporary, until merged)
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

parm/config/gfs/config.esfc

@@ -23,8 +23,9 @@ fi
 
 # set up soil analysis
 if [[ ${GSI_SOILANAL} = "YES" ]]; then
-    export DO_LNDINC=".true."
-    export LND_SOI_FILE="lnd_incr"
+     export DO_SOI_INC=".true."
+     export GCYCLE_INTERP_LNDINC=".false."
+     export LSOIL_INCR=2
 fi
 
 echo "END: config.esfc"

parm/config/gfs/config.resources

@@ -679,12 +679,12 @@ case ${step} in
         ntasks_gfs=825
         threads_per_task=5
         ;;
-      "C384")
+      "C384" | "C192")
         ntasks_gdas=160
         ntasks_gfs=160
         threads_per_task=10
         ;;
-      "C192" | "C96" | "C48")
+      "C96" | "C48")
         ntasks_gdas=84
         ntasks_gfs=84
         threads_per_task=5
@@ -1234,7 +1234,7 @@ case ${step} in
     ;;
 
   "esfc")
-    walltime="00:25:00"
+    walltime="01:00:00"
     ntasks=80
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))

parm/config/gfs/config.sfcanl

@@ -12,5 +12,11 @@ echo "BEGIN: config.sfcanl"
 if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
    export DONST="NO"
 fi
+if [[ "${GSI_SOILANAL}" = "YES" ]]; then
+    export DO_LNDINC=".true."
+    export DO_SOI_INC=".true."
+    export GCYCLE_INTERP_LNDINC=".false."
+    export LSOIL_INCR=2
+fi
 
 echo "END: config.sfcanl"

scripts/exgdas_enkf_sfc.sh

@@ -65,6 +65,8 @@ export CYCLVARS=${CYCLVARS:-"FSNOL=-2.,FSNOS=99999.,"}
 export FHOUR=${FHOUR:-0}
 export DELTSFC=${DELTSFC:-6}
 
+REGRIDSH=${REGRIDSH:-${USHgfs}/regrid_gsiSfcIncr_to_tile.sh}
+
 APRUN_ESFC=${APRUN_ESFC:-${APRUN:-""}}
 NTHREADS_ESFC=${NTHREADS_ESFC:-${NTHREADS:-1}}
 
@@ -115,6 +117,23 @@ else
     export NST_FILE="NULL"
 fi
 
+# regrid the surface increment files
+if [[ ${GSI_SOILANAL} = "YES" ]]; then
+
+    export CASE_IN=${CASE_ENS}
+    export CASE_OUT=${CASE_ENS}
+    export OCNRES_OUT=${OCNRES}
+    export NMEM_REGRID=${NMEM_ENS}
+    if [ $DOIAU = "YES" ]; then
+        export LFHR=3 # match BDATE
+    else # DOSFCANL_ENKF
+        export LFHR=6 # PDYcyc
+    fi
+
+    $REGRIDSH
+
+fi
+
 export APRUNCY=${APRUN_CYCLE:-$APRUN_ESFC}
 export OMP_NUM_THREADS_CY=${NTHREADS_CYCLE:-$NTHREADS_ESFC}
 export MAX_TASKS_CY=$NMEM_ENS
@@ -144,7 +163,7 @@ if [ $DOIAU = "YES" ]; then
                 COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL
 
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
-                COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+                COMIN_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
                 COMIN_SNOW_ANALYSIS_MEM:COM_SNOW_ANALYSIS_TMPL
@@ -164,10 +183,11 @@ if [ $DOIAU = "YES" ]; then
             ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}"
 
             if [[ ${GSI_SOILANAL} = "YES" ]]; then
-                FHR=6
-                ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
-                   "${DATA}/lnd_incr.${cmem}"
+                FHR=${LFHR}
+                 ${NCP} "${COMIN_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" \
+                   "${DATA}/soil_xainc.${cmem}" 
             fi
+
         done # ensembles
 
         CDATE="${PDY}${cyc}" ${CYCLESH}
@@ -186,18 +206,9 @@ if [ $DOIAU = "YES" ]; then
             MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
                 COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL
 
-            MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
-                COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
-
             [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}"
             cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc"
 
-
-            if [[ ${GSI_SOILANAL} = "YES" ]]; then
-                FHR=6
-                ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
-                   "${DATA}/lnd_incr.${cmem}"
-            fi
         done # ensembles
 
     done
@@ -225,6 +236,9 @@ if [ $DOSFCANL_ENKF = "YES" ]; then
             RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${gPDY} HH=${gcyc} declare_from_tmpl \
                 COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL
 
+            MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+                COMIN_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
             # determine where the input snow restart files come from
             if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then
                 sfcdata_dir="${COMIN_SNOW_ANALYSIS_MEM}"
@@ -238,6 +252,11 @@ if [ $DOSFCANL_ENKF = "YES" ]; then
             ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc"      "${DATA}/fngrid.${cmem}"
             ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}"
 
+            if [[ ${GSI_SOILANAL} = "YES" ]]; then
+                FHR=${LFHR}
+                 ${NCP} "${COMIN_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" \
+                   "${DATA}/soil_xainc.${cmem}" 
+            fi
         done
 
         CDATE="${PDY}${cyc}" ${CYCLESH}

scripts/exgdas_enkf_update.sh

@@ -213,7 +213,7 @@ for imem in $(seq 1 $NMEM_ENS); do
    for FHR in $nfhrs; do
       ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}atmf00${FHR}${ENKF_SUFFIX}.nc" \
          "sfg_${PDY}${cyc}_fhr0${FHR}_${memchar}"
-      if [ $GSI_SOILANAL = "YES" ]; then
+      if [[ $GSI_SOILANAL = "YES" ]]; then
          ${NLN} "${COM_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}sfcf00${FHR}${ENKF_SUFFIX}.nc" \
              "bfg_${PDY}${cyc}_fhr0${FHR}_${memchar}"
       fi
@@ -238,7 +238,7 @@ for imem in $(seq 1 $NMEM_ENS); do
                "incr_${PDY}${cyc}_fhr0${FHR}_${memchar}"
          fi
       fi
-      if [ $GSI_SOILANAL = "YES" ]; then
+      if [[ ${GSI_SOILANAL} = "YES" ]]; then
           ${NLN} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX}sfci00${FHR}.nc" \
            "sfcincr_${PDY}${cyc}_fhr0${FHR}_${memchar}"
       fi
@@ -250,10 +250,16 @@ for FHR in $nfhrs; do
 
    ${NLN} "${COM_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}atmf00${FHR}.ensmean.nc" \
       "sfg_${PDY}${cyc}_fhr0${FHR}_ensmean"
-   if [ $cnvw_option = ".true." ]; then
+   if [[ $cnvw_option = ".true." ]]; then
       ${NLN} "${COM_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}sfcf00${FHR}.ensmean.nc" \
          "sfgsfc_${PDY}${cyc}_fhr0${FHR}_ensmean"
    fi
+   if [[ ${GSI_SOILANAL} = "YES" ]]; then
+      ${NLN} "${COM_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}sfcf00${FHR}.ensmean.nc" \
+         "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean"
+      ${NLN} "${COM_ATMOS_ANALYSIS_STAT}/${APREFIX}sfci00${FHR}.nc" \
+         "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean"
+   fi
 done
 
 if [[ $USE_CFP = "YES" ]]; then