From 9afb1bf2d7a1a3fa0ea8ac04627238dc798cd299 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 9 Oct 2025 16:24:15 +0000
Subject: [PATCH 01/28] Initial commit

---
 dev/parm/config/gfs/config.resources |   7 +-
 sorc/ufs_utils.fd                    |   2 +-
 ush/regrid_gsiSfcIncr_to_tile.sh     | 137 ++++++++++++++++++---------
 3 files changed, 97 insertions(+), 49 deletions(-)

diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources
index 3b378af0d18..0f93f5089df 100644
--- a/dev/parm/config/gfs/config.resources
+++ b/dev/parm/config/gfs/config.resources
@@ -1317,9 +1317,12 @@ case ${step} in
 
   "esfc")
     walltime="01:15:00"
-    ntasks=80
+    ntasks=$(( NMEM_ENS * 6))
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    node_frac = 0.5
+    tasks_per_node=$(( floor(max_tasks_per_node*node_frac/6)*6 ))
+    #tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+
     threads_per_task_cycle=${threads_per_task}
     tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle ))
     ;;
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index ae647342264..5d0c013b17e 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit ae64734226402fcb438ce4feed5a43829b47342e
+Subproject commit 5d0c013b17ed4fa01f8e084627a4611b1dd78a19
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 891a46ad9f3..7cf6cfa12de 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -16,7 +16,6 @@ export REDERR=${REDERR:-'2>'}
 export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
-NMEM_REGRID=${NMEM_REGRID:-1}
 CASE_IN=${CASE_IN:-${CASE_ENS}}
 LFHR=${LFHR:-6}
 
@@ -44,16 +43,33 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     IFS=',' read -ra landifhrs <<< "${IAUFHRS}"
 fi
 export in_fname="'enkfgdas.sfci'"
-export out_fname="'sfci'"
-export dir_mask_in="'./'"
+export dir_coord_in="'${DATA}/'"
+export dir_coord_out="'${DATA}/'"
+export dir_mask_in="'${DATA}/'"
+export dir_mask_out="'${DATA}/'"
 export fname_mask_in="'NULL'"
 export ires=${LONB_CASE_IN}
 export jres=${LATB_CASE_IN}
 export ireso=${CASE_OUT:1}
 export jreso=${CASE_OUT:1}
+export n_data="${NMEM_ENS}"
 
 regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" 
 
+if (( LFHR >= 0 )); then
+    soilinc_fhrs=("${LFHR}")
+else # construct restart times for deterministic member
+    soilinc_fhrs=("${assim_freq}") # increment file at middle of window
+    if [[ "${DOIAU:-}" == "YES" ]]; then  # Update surface restarts at beginning of window
+        half_window=$(( assim_freq / 2 ))
+        soilinc_fhrs+=("${half_window}")
+    fi
+fi
+
+#
+# Stage input files
+#
+
 # input, fixed files
 cpreq "${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc" \
       "${DATA}/gaussian_scrip.nc"
@@ -69,21 +85,17 @@ for n in $(seq 1 "${ntiles}"); do
           "${DATA}/${CASE_OUT}_grid.tile${n}.nc"
 done
 
-if (( LFHR >= 0 )); then
-    soilinc_fhrs=("${LFHR}")
-else # construct restart times for deterministic member
-    soilinc_fhrs=("${assim_freq}") # increment file at middle of window
-    if [[ "${DOIAU:-}" == "YES" ]]; then  # Update surface restarts at beginning of window
-        half_window=$(( assim_freq / 2 ))
-        soilinc_fhrs+=("${half_window}")
-    fi
-fi
+#export in_dir="("
+export in_dir=""
+for imem in $(seq 1 "${NMEM_ENS}"); do
+    cmem=$(printf %03i "${imem}")
+    memchar="mem${cmem}"
 
-for imem in $(seq 1 "${NMEM_REGRID}"); do
-    if (( NMEM_REGRID > 1 )); then
-        cmem=$(printf %03i "${imem}")
-        memchar="mem${cmem}"
+    # Create run directory for this member
+    memdir="${DATA}/${memchar}"
+    mkdir -p "${memdir}"
 
+    if (( NMEM_ENS > 1 )); then
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
@@ -91,51 +103,84 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
             COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
     fi
 
+    # Append to input directory list
+    if [ $imem -gt 1 ]; then
+        in_dir+=", "
+    fi
+    in_dir+="\"${memdir}\""
+
     for FHR in "${soilinc_fhrs[@]}"; do
+        cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
+               "${memdir}/enkfgdas.sfci00${FHR}.nc"
+    done 
 
-        export add_time_dim=".false."
-        export time_list="${FHR}"
+    if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
+        for FHI in "${landifhrs[@]}"; do
+            cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHI}.nc" \
+                  "${memdir}/enkfgdas.sfci00${FHI}.nc"
+        done
+    fi
+done
 
-        rm -f "regrid.nml"
-        atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+# Finish defining input/output directory list
+#in_dir+=")"
+export out_dir="${in_dir}"
 
-        cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
-               "${DATA}/enkfgdas.sfci00${FHR}.nc"
+#
+# Regrid soil increments and save to COMOUT
+#
 
-        ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+for FHR in "${soilinc_fhrs[@]}"; do
+    export add_time_dim=".false."
+    export time_list="${FHR}"
+    export out_fname="'sfci00${FHR}'"
 
-        for n in $(seq 1 "${ntiles}"); do
-            cpfs "${DATA}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"
-        done
-    done 
+    rm -f "regrid.nml"
+    atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
-    if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
+    ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+	export err=$?
+	if [[ ${err} -ne 0 ]]; then
+	    err_exit "${REGRID_EXEC} failed, ABORT!"
+	fi
+done
 
-        export add_time_dim=".true."
-        export time_list="${IAUFHRS}"
+if [[ "${DO_LAND_IAU}" = ".true." ]]; then
+    export add_time_dim=".true."
+    export time_list="${IAUFHRS}"
+    export out_fname="'sfci'"
+
+    rm -f "regrid.nml"
+    atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+
+    export pgm="${REGRID_EXEC}"
+	${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+	export err=$?
+	if [[ ${err} -ne 0 ]]; then
+	    err_exit "${pgm} failed, ABORT!"
+	fi
+fi
 
-        rm -f "regrid.nml"
-        atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+#
+# Save regridded files to COMOUT
+#
 
-        for FHI in "${landifhrs[@]}"; do
-            cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHI}.nc" \
-                  "${DATA}/enkfgdas.sfci00${FHI}.nc"
+for imem in $(seq 1 "${NMEM_ENS}"); do
+    cmem=$(printf %03i "${imem}")
+    memchar="mem${cmem}"
+    memdir="${DATA}/${memchar}"
+
+    for FHR in "${soilinc_fhrs[@]}"; do
+        for n in $(seq 1 "${ntiles}"); do
+            cpfs "${memdir}/sfci00${FHR}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"
         done
-        
-        export pgm="${REGRID_EXEC}"
-	      ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
-	      export err=$?
-	      if [[ ${err} -ne 0 ]]; then
-	          err_exit "${pgm} failed, ABORT!"
-	      fi
+    done
 
+    if [[ "${DO_LAND_IAU}" = ".true." ]]; then
         for n in $(seq 1 "${ntiles}"); do
-            cpfs "${DATA}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc"
+            cpfs "${memdir}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc"
         done
-	    
     fi
-
 done
 
 exit 0
-

From 694bcd0136d9ee79f2c23d48776eb69fecd5868f Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 9 Oct 2025 16:26:06 +0000
Subject: [PATCH 02/28] Update ntasks_regrid

---
 env/HERA.env | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/env/HERA.env b/env/HERA.env
index a2177cf0986..07c20c87ebe 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -302,9 +302,8 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks} "
 
 elif [[ "${step}" = "epos" ]]; then
 

From 5e367cf042f5a6a558901de648c86ca3dda1e684 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 12:45:28 +0000
Subject: [PATCH 03/28] Debug

---
 scripts/exgdas_enkf_sfc.sh       |  1 -
 sorc/ufs_utils.fd                |  2 +-
 ush/regrid_gsiSfcIncr_to_tile.sh | 13 ++++++++++---
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
index e3b7c5ec6f2..bf251978aa5 100755
--- a/scripts/exgdas_enkf_sfc.sh
+++ b/scripts/exgdas_enkf_sfc.sh
@@ -136,7 +136,6 @@ if [[ "${DO_GSISOILDA}" == "YES" ]]; then
     export CASE_IN=${CASE_ENS}
     export CASE_OUT=${CASE_ENS}
     export OCNRES_OUT=${OCNRES}
-    export NMEM_REGRID=${NMEM_ENS}
     if [[ "${DOIAU}" == "YES" ]]; then
         export LFHR=3 # match BDATE
     else # DOSFCANL_ENKF
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index 5d0c013b17e..a4434295c35 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit 5d0c013b17ed4fa01f8e084627a4611b1dd78a19
+Subproject commit a4434295c35c9f1be73e75f2ec466a3a52358d9c
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 7cf6cfa12de..0bc30d422c8 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -52,7 +52,7 @@ export ires=${LONB_CASE_IN}
 export jres=${LATB_CASE_IN}
 export ireso=${CASE_OUT:1}
 export jreso=${CASE_OUT:1}
-export n_data="${NMEM_ENS}"
+export nmem_regrid="${NMEM_ENS}"
 
 regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" 
 
@@ -85,14 +85,13 @@ for n in $(seq 1 "${ntiles}"); do
           "${DATA}/${CASE_OUT}_grid.tile${n}.nc"
 done
 
-#export in_dir="("
 export in_dir=""
 for imem in $(seq 1 "${NMEM_ENS}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
 
     # Create run directory for this member
-    memdir="${DATA}/${memchar}"
+    memdir="./${memchar}"
     mkdir -p "${memdir}"
 
     if (( NMEM_ENS > 1 )); then
@@ -170,6 +169,14 @@ for imem in $(seq 1 "${NMEM_ENS}"); do
     memchar="mem${cmem}"
     memdir="${DATA}/${memchar}"
 
+    if (( NMEM_ENS > 1 )); then
+        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+            COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+            COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+    fi
+
     for FHR in "${soilinc_fhrs[@]}"; do
         for n in $(seq 1 "${ntiles}"); do
             cpfs "${memdir}/sfci00${FHR}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"

From e2a1f847d52c74601cad4d7b98fc4f1776c4e6c3 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 12:47:40 +0000
Subject: [PATCH 04/28] Debug

---
 dev/parm/config/gfs/config.resources | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources
index 0f93f5089df..754e4258da5 100644
--- a/dev/parm/config/gfs/config.resources
+++ b/dev/parm/config/gfs/config.resources
@@ -1319,9 +1319,8 @@ case ${step} in
     walltime="01:15:00"
     ntasks=$(( NMEM_ENS * 6))
     threads_per_task=1
-    node_frac = 0.5
-    tasks_per_node=$(( floor(max_tasks_per_node*node_frac/6)*6 ))
-    #tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    node_frac=2
+    tasks_per_node=$(( max_tasks_per_node/(node_frac*6)*6 ))
 
     threads_per_task_cycle=${threads_per_task}
     tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle ))

From b5586d7856cc283df7b798a346bebc64f2a1d222 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 17:42:29 +0000
Subject: [PATCH 05/28] Debug

---
 scripts/exgdas_enkf_sfc.sh | 1 +
 sorc/ufs_utils.fd          | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
index bf251978aa5..e3b7c5ec6f2 100755
--- a/scripts/exgdas_enkf_sfc.sh
+++ b/scripts/exgdas_enkf_sfc.sh
@@ -136,6 +136,7 @@ if [[ "${DO_GSISOILDA}" == "YES" ]]; then
     export CASE_IN=${CASE_ENS}
     export CASE_OUT=${CASE_ENS}
     export OCNRES_OUT=${OCNRES}
+    export NMEM_REGRID=${NMEM_ENS}
     if [[ "${DOIAU}" == "YES" ]]; then
         export LFHR=3 # match BDATE
     else # DOSFCANL_ENKF
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index a4434295c35..2df84726b9f 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit a4434295c35c9f1be73e75f2ec466a3a52358d9c
+Subproject commit 2df84726b9fea3aeb375b2d69a93e085fe083af3

From afad1da044ba65c57c39fa5b305d5bdfb4df53e4 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 17:44:07 +0000
Subject: [PATCH 06/28] update

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 0bc30d422c8..79dc65ba458 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -43,16 +43,13 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     IFS=',' read -ra landifhrs <<< "${IAUFHRS}"
 fi
 export in_fname="'enkfgdas.sfci'"
-export dir_coord_in="'${DATA}/'"
-export dir_coord_out="'${DATA}/'"
 export dir_mask_in="'${DATA}/'"
-export dir_mask_out="'${DATA}/'"
 export fname_mask_in="'NULL'"
 export ires=${LONB_CASE_IN}
 export jres=${LATB_CASE_IN}
 export ireso=${CASE_OUT:1}
 export jreso=${CASE_OUT:1}
-export nmem_regrid="${NMEM_ENS}"
+NMEM_REGRID=${NMEM_REGRID:-1}
 
 regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" 
 
@@ -86,20 +83,21 @@ for n in $(seq 1 "${ntiles}"); do
 done
 
 export in_dir=""
-for imem in $(seq 1 "${NMEM_ENS}"); do
+for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
 
-    # Create run directory for this member
-    memdir="./${memchar}"
-    mkdir -p "${memdir}"
-
-    if (( NMEM_ENS > 1 )); then
+    if (( NMEM_REGRID > 1 )); then
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        memdir="${DATA}/${memchar}"
+        mkdir -p "${memdir}"
+    else
+        memdir="${DATA}"
     fi
 
     # Append to input directory list
@@ -164,17 +162,20 @@ fi
 # Save regridded files to COMOUT
 #
 
-for imem in $(seq 1 "${NMEM_ENS}"); do
+for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
-    memdir="${DATA}/${memchar}"
 
-    if (( NMEM_ENS > 1 )); then
+    if (( NMEM_REGRID > 1 )); then
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        memdir="${DATA}/${memchar}"
+    else
+        memdir="${DATA}"
     fi
 
     for FHR in "${soilinc_fhrs[@]}"; do

From ff22f7c2da6dfe83c2d16fe0020f9d7d0be60e0d Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 17:48:36 +0000
Subject: [PATCH 07/28] Update

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 79dc65ba458..1894746e2d4 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -43,7 +43,9 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     IFS=',' read -ra landifhrs <<< "${IAUFHRS}"
 fi
 export in_fname="'enkfgdas.sfci'"
-export dir_mask_in="'${DATA}/'"
+export out_fname="'sfci'"
+export in_dir=""
+export dir_mask_in="'./'"
 export fname_mask_in="'NULL'"
 export ires=${LONB_CASE_IN}
 export jres=${LATB_CASE_IN}
@@ -82,7 +84,6 @@ for n in $(seq 1 "${ntiles}"); do
           "${DATA}/${CASE_OUT}_grid.tile${n}.nc"
 done
 
-export in_dir=""
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"

From cc30e68f6aa5423cd9363413761a5bf224bcc427 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 14 Oct 2025 17:51:54 +0000
Subject: [PATCH 08/28] Update

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 1894746e2d4..75f1fb8b3b0 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -16,6 +16,7 @@ export REDERR=${REDERR:-'2>'}
 export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
+NMEM_REGRID=${NMEM_REGRID:-1}
 CASE_IN=${CASE_IN:-${CASE_ENS}}
 LFHR=${LFHR:-6}
 
@@ -51,7 +52,6 @@ export ires=${LONB_CASE_IN}
 export jres=${LATB_CASE_IN}
 export ireso=${CASE_OUT:1}
 export jreso=${CASE_OUT:1}
-NMEM_REGRID=${NMEM_REGRID:-1}
 
 regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" 
 

From c4a22a55c77d429978245358315c0944be07e2b7 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Wed, 15 Oct 2025 12:50:30 +0000
Subject: [PATCH 09/28] Debug

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 75f1fb8b3b0..f7ea5b77fe2 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -97,15 +97,16 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
 
         memdir="${DATA}/${memchar}"
         mkdir -p "${memdir}"
+
+        if [ $imem -gt 1 ]; then
+            in_dir+=", "
+        fi
+        in_dir+="\"./${memchar}/\""        
     else
         memdir="${DATA}"
-    fi
 
-    # Append to input directory list
-    if [ $imem -gt 1 ]; then
-        in_dir+=", "
+        in_dir="'./'"
     fi
-    in_dir+="\"${memdir}\""
 
     for FHR in "${soilinc_fhrs[@]}"; do
         cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
@@ -121,7 +122,6 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
 done
 
 # Finish defining input/output directory list
-#in_dir+=")"
 export out_dir="${in_dir}"
 
 #

From 6d45be94aa7cf15e07d3a165d17879a5c91ce943 Mon Sep 17 00:00:00 2001
From: "catherine.thomas" <catherine.thomas@noaa.gov>
Date: Wed, 22 Oct 2025 16:16:18 +0000
Subject: [PATCH 10/28] regrid updates for IAU and WCOSS2

---
 env/WCOSS2.env                   |  3 ++-
 scripts/exgdas_enkf_sfc.sh       | 12 ++++++------
 ush/regrid_gsiSfcIncr_to_tile.sh | 32 ++++++++++++++++++--------------
 3 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index 1734968ee51..d9c6578a940 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -323,7 +323,8 @@ elif [[ "${step}" = "esfc" ]]; then
 
     # REGRID requires 6 tasks for reproducibility
     ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    export APRUN_REGRID="${launcher} -n ${ntasks} "
+    #export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
 
 elif [[ "${step}" = "epos" ]]; then
 
diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
index e3b7c5ec6f2..e725a6d8dfa 100755
--- a/scripts/exgdas_enkf_sfc.sh
+++ b/scripts/exgdas_enkf_sfc.sh
@@ -163,7 +163,7 @@ if [[ "$DOIAU" == "YES" ]]; then
 
         export TILE_NUM=$n
 
-        # Copy inputs from COMIN to DATA
+	# Copy inputs from COMIN to DATA
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
             if (( smem > NMEM_ENS_MAX )); then
@@ -201,19 +201,19 @@ if [[ "$DOIAU" == "YES" ]]; then
             cpreq "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc"     "${DATA}/fngrid.${cmem}"
             cpreq "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}"
 
-            if [[ "${DO_GSISOILDA}" == "YES" ]]; then
+            if [[ "${DO_GSISOILDA}" == "YES" && "${DO_LAND_IAU}" == ".false." ]]; then
                  cpreq "${COMIN_ATMOS_ANALYSIS_MEM}/sfci00${LFHR}.tile${n}.nc" \
                    "${DATA}/soil_xainc.${cmem}"
             fi
 
         done # ensembles
-
-        "${CYCLESH}" && true
+         
+	"${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
            err_exit "Failed to update surface fields!"
         fi
-
+        
         # Copy outputs from DATA to COMOUT
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
@@ -281,7 +281,7 @@ if [[ "${DOSFCANL_ENKF}" == "YES" ]]; then
                    "${DATA}/soil_xainc.${cmem}"
             fi
         done
-
+        
         "${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index f7ea5b77fe2..d1121680580 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -128,20 +128,22 @@ export out_dir="${in_dir}"
 # Regrid soil increments and save to COMOUT
 #
 
-for FHR in "${soilinc_fhrs[@]}"; do
-    export add_time_dim=".false."
-    export time_list="${FHR}"
-    export out_fname="'sfci00${FHR}'"
+if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
+    for FHR in "${soilinc_fhrs[@]}"; do
+        export add_time_dim=".false."
+        export time_list="${FHR}"
+        export out_fname="'sfci00${FHR}'"
 
-    rm -f "regrid.nml"
-    atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+        rm -f "regrid.nml"
+        atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
-    ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
-	export err=$?
+        ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+    	export err=$?
 	if [[ ${err} -ne 0 ]]; then
 	    err_exit "${REGRID_EXEC} failed, ABORT!"
 	fi
-done
+    done
+fi
 
 if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     export add_time_dim=".true."
@@ -179,15 +181,17 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
         memdir="${DATA}"
     fi
 
-    for FHR in "${soilinc_fhrs[@]}"; do
-        for n in $(seq 1 "${ntiles}"); do
-            cpfs "${memdir}/sfci00${FHR}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"
+    if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
+        for FHR in "${soilinc_fhrs[@]}"; do
+            for n in $(seq 1 "${ntiles}"); do
+                cpfs "${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"
+            done
         done
-    done
+    fi
 
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then
         for n in $(seq 1 "${ntiles}"); do
-            cpfs "${memdir}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc"
+            cpfs "${memdir}/sfci.mem${imem}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc"
         done
     fi
 done

From e79c59ecb080521f1960fc043abd2148b8ee8670 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 23 Oct 2025 15:09:44 +0000
Subject: [PATCH 11/28] ufs utils fix

---
 sorc/ufs_utils.fd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index 2df84726b9f..f091f4ec024 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit 2df84726b9fea3aeb375b2d69a93e085fe083af3
+Subproject commit f091f4ec02459d9d20052264aa7cca31d633f259

From 849255906e9bb4536893990c0a62eb961096db80 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 23 Oct 2025 15:11:13 +0000
Subject: [PATCH 12/28] use mpmpd to move files around

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 70 +++++++++++++++++++++++++-------
 1 file changed, 56 insertions(+), 14 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index d1121680580..8e42475a407 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -69,21 +69,34 @@ fi
 # Stage input files
 #
 
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 744 cmdfile
+
+# Create MDMP command file for fixed files
+rm -f cmdfile.0
+touch cmdfile.0
+chmod 744 cmdfile.0
+
 # input, fixed files
-cpreq "${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc" \
-      "${DATA}/gaussian_scrip.nc"
+echo "cpreq ${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc \
+            ${DATA}/gaussian_scrip.nc" >> cmdfile.0
 
 # output, fixed files
-cpreq "${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc" \
-      "${DATA}/${CASE_OUT}_mosaic.nc"
+echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc \
+            ${DATA}/${CASE_OUT}_mosaic.nc" >> cmdfile.0
 
 for n in $(seq 1 "${ntiles}"); do
-    cpreq "${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc" \
-          "${DATA}/vegetation_type.tile${n}.nc"
-    cpreq "${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc" \
-          "${DATA}/${CASE_OUT}_grid.tile${n}.nc"
+    echo "cpreq ${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc \
+                ${DATA}/vegetation_type.tile${n}.nc" >> cmdfile.0
+    echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc \
+                ${DATA}/${CASE_OUT}_grid.tile${n}.nc" >> cmdfile.0
 done
 
+# Append fixed files command file to master command file
+"cmdfile.0" >> cmdfile
+
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
@@ -108,19 +121,30 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
         in_dir="'./'"
     fi
 
+    # Create MPMD command file for this member
+    rm -f "cmdfile.${imem}"
+    touch "cmdfile.${imem}"
+    chmod 744 "cmdfile.${imem}"
+
     for FHR in "${soilinc_fhrs[@]}"; do
-        cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \
-               "${memdir}/enkfgdas.sfci00${FHR}.nc"
+        echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc \
+                    ${memdir}/enkfgdas.sfci00${FHR}.nc" >> "cmdfile.${imem}"
     done 
 
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
         for FHI in "${landifhrs[@]}"; do
-            cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHI}.nc" \
-                  "${memdir}/enkfgdas.sfci00${FHI}.nc"
+            echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHI}.nc \
+                        ${memdir}/enkfgdas.sfci00${FHI}.nc" >> "cmdfile.${imem}"
         done
     fi
+
+    # Append this member's command file to master command file
+    "cmdfile.${imem}" >> cmdfile
 done
 
+# Run MPMD to stage input files
+${APRUN_REGRID} cmdfile
+
 # Finish defining input/output directory list
 export out_dir="${in_dir}"
 
@@ -165,6 +189,11 @@ fi
 # Save regridded files to COMOUT
 #
 
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 744 cmdfile
+
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
@@ -181,19 +210,32 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
         memdir="${DATA}"
     fi
 
+    # Create MPMD command file for this member
+    rm -f "cmdfile.${imem}"
+    touch "cmdfile.${imem}"
+    chmod 744 "cmdfile.${imem}"
+
     if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
         for FHR in "${soilinc_fhrs[@]}"; do
             for n in $(seq 1 "${ntiles}"); do
-                cpfs "${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc"
+                echo "cpfs ${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \
+                      ${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" >> "cmdfile.${imem}"
             done
         done
     fi
 
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then
         for n in $(seq 1 "${ntiles}"); do
-            cpfs "${memdir}/sfci.mem${imem}.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc"
+            echo "cpfs ${memdir}/sfci.mem${imem}.tile${n}.nc \
+                  ${COMOUT_ATMOS_ANALYSIS_MEM}/sfc_inc.tile${n}.nc" >> "cmdfile.${imem}"
         done
     fi
+
+    # Append this member's command file to master command file
+    "cmdfile.${imem}" >> cmdfile
 done
 
+# Run MPMD to save output files
+${APRUN_REGRID} cmdfile
+
 exit 0

From 1a8b950493c2072e68d8231fb510713e90128a4b Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Mon, 27 Oct 2025 14:23:58 +0000
Subject: [PATCH 13/28] Debug

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 38 +++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 8e42475a407..820696e23cb 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -16,6 +16,13 @@ export REDERR=${REDERR:-'2>'}
 export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
+# Use CFP for ensemble job
+if [[ "$RUN" == *"enkf"* ]]; then
+    export USE_CFP=YES
+else
+    export USE_CFP=NO
+fi
+
 NMEM_REGRID=${NMEM_REGRID:-1}
 CASE_IN=${CASE_IN:-${CASE_ENS}}
 LFHR=${LFHR:-6}
@@ -72,12 +79,13 @@ fi
 # Create master MPMD command file
 rm -f cmdfile
 touch cmdfile
-chmod 744 cmdfile
+chmod 755 cmdfile
 
 # Create MDMP command file for fixed files
 rm -f cmdfile.0
 touch cmdfile.0
-chmod 744 cmdfile.0
+chmod 755 cmdfile.0
+echo "#!/bin/bash" > cmdfile.0
 
 # input, fixed files
 echo "cpreq ${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc \
@@ -95,7 +103,7 @@ for n in $(seq 1 "${ntiles}"); do
 done
 
 # Append fixed files command file to master command file
-"cmdfile.0" >> cmdfile
+echo "${DATA}/cmdfile.0" >> cmdfile
 
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
@@ -124,7 +132,8 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     # Create MPMD command file for this member
     rm -f "cmdfile.${imem}"
     touch "cmdfile.${imem}"
-    chmod 744 "cmdfile.${imem}"
+    chmod 755 "cmdfile.${imem}"
+    echo "#!/bin/bash" > "cmdfile.${imem}"
 
     for FHR in "${soilinc_fhrs[@]}"; do
         echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc \
@@ -139,11 +148,15 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     fi
 
     # Append this member's command file to master command file
-    "cmdfile.${imem}" >> cmdfile
+    echo "${DATA}/cmdfile.${imem}" >> cmdfile
 done
 
 # Run MPMD to stage input files
-${APRUN_REGRID} cmdfile
+"${USHgfs}/run_mpmd.sh" "cmdfile" && true
+export err=$?
+if [[ ${err} -ne 0 ]]; then
+    err_exit "run_mpmd.sh failed!"
+fi
 
 # Finish defining input/output directory list
 export out_dir="${in_dir}"
@@ -192,7 +205,7 @@ fi
 # Create master MPMD command file
 rm -f cmdfile
 touch cmdfile
-chmod 744 cmdfile
+chmod 755 cmdfile
 
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
@@ -213,7 +226,8 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     # Create MPMD command file for this member
     rm -f "cmdfile.${imem}"
     touch "cmdfile.${imem}"
-    chmod 744 "cmdfile.${imem}"
+    chmod 755 "cmdfile.${imem}"
+    echo "#!/bin/bash" > "cmdfile.${imem}"
 
     if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
         for FHR in "${soilinc_fhrs[@]}"; do
@@ -232,10 +246,14 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     fi
 
     # Append this member's command file to master command file
-    "cmdfile.${imem}" >> cmdfile
+    echo "${DATA}/cmdfile.${imem}" >> cmdfile
 done
 
 # Run MPMD to save output files
-${APRUN_REGRID} cmdfile
+"${USHgfs}/run_mpmd.sh" "cmdfile" && true
+export err=$?
+if [[ ${err} -ne 0 ]]; then
+    err_exit "run_mpmd.sh failed!"
+fi
 
 exit 0

From c26110831e56b407c3e387ee77d55de1cf2c6996 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Wed, 29 Oct 2025 15:41:53 +0000
Subject: [PATCH 14/28] Shell norms

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index fc4d74d45c1..e7afecead60 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -17,7 +17,7 @@ export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
 # Use CFP for ensemble job
-if [[ "$RUN" == *"enkf"* ]]; then
+if [[ "${RUN}" == *"enkf"* ]]; then
     export USE_CFP=YES
 else
     export USE_CFP=NO
@@ -119,7 +119,7 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
         memdir="${DATA}/${memchar}"
         mkdir -p "${memdir}"
 
-        if [ $imem -gt 1 ]; then
+        if [[ "${imem}" -gt 1 ]]; then
             in_dir+=", "
         fi
         in_dir+="\"./${memchar}/\""        

From 8a022dcba46e06bad73b69345a32b566c497a063 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Wed, 29 Oct 2025 17:25:40 +0000
Subject: [PATCH 15/28] shellnorms

---
 dev/parm/config/gfs/config.resources | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources
index dfcaecb96fb..b68ed76054a 100644
--- a/dev/parm/config/gfs/config.resources
+++ b/dev/parm/config/gfs/config.resources
@@ -1319,8 +1319,9 @@ case ${step} in
     walltime="01:15:00"
     ntasks=$(( NMEM_ENS * 6))
     threads_per_task=1
-    node_frac=2
-    tasks_per_node=$(( max_tasks_per_node/(node_frac*6)*6 ))
+    node_numerator=1
+    node_denominator=2
+    tasks_per_node=$(( node_numerator*max_tasks_per_node/node_denominator ))
 
     threads_per_task_cycle=${threads_per_task}
     tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle ))

From 99352b9e7ad7afd3069210b137d026d0dbe470dd Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 30 Oct 2025 13:33:29 +0000
Subject: [PATCH 16/28] Update all env files

---
 env/AWSPW.env    | 3 +++
 env/AZUREPW.env  | 3 +++
 env/GAEAC5.env   | 5 ++---
 env/GAEAC6.env   | 5 ++---
 env/GOOGLEPW.env | 3 +++
 env/HERA.env     | 2 +-
 env/HERCULES.env | 5 ++---
 env/ORION.env    | 5 ++---
 env/URSA.env     | 5 ++---
 env/WCOSS2.env   | 6 ++----
 10 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/env/AWSPW.env b/env/AWSPW.env
index 1c2f903f28b..e974d109cd0 100755
--- a/env/AWSPW.env
+++ b/env/AWSPW.env
@@ -107,6 +107,9 @@ elif [[ "${step}" = "esfc" ]]; then
     export NTHREADS_CYCLE=${threads_per_task_cycle:-1}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
+
 elif [[ "${step}" = "upp" ]]; then
 
     export NTHREADS_UPP=1
diff --git a/env/AZUREPW.env b/env/AZUREPW.env
index 11a45c78ffd..f407b19302b 100755
--- a/env/AZUREPW.env
+++ b/env/AZUREPW.env
@@ -107,6 +107,9 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
diff --git a/env/GAEAC5.env b/env/GAEAC5.env
index b6da2dd1c4f..83813f447c5 100755
--- a/env/GAEAC5.env
+++ b/env/GAEAC5.env
@@ -285,9 +285,8 @@ case ${step} in
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
  ;;
  "epos")
diff --git a/env/GAEAC6.env b/env/GAEAC6.env
index 31182f89da4..b37572ba7cf 100755
--- a/env/GAEAC6.env
+++ b/env/GAEAC6.env
@@ -317,9 +317,8 @@ case ${step} in
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
  ;;
  "epos")
diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env
index b6a4d8c01f7..9a821023e3f 100755
--- a/env/GOOGLEPW.env
+++ b/env/GOOGLEPW.env
@@ -106,6 +106,9 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
diff --git a/env/HERA.env b/env/HERA.env
index 07c20c87ebe..57aefcc66af 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -303,7 +303,7 @@ elif [[ "${step}" = "esfc" ]]; then
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
     # REGRID requires 6*NMEM_ENS tasks for reproducibility
-    export APRUN_REGRID="${launcher} -n ${ntasks} "
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
 elif [[ "${step}" = "epos" ]]; then
 
diff --git a/env/HERCULES.env b/env/HERCULES.env
index 806714e38f1..46a6db49216 100755
--- a/env/HERCULES.env
+++ b/env/HERCULES.env
@@ -322,9 +322,8 @@ case ${step} in
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
  ;;
  "epos")
diff --git a/env/ORION.env b/env/ORION.env
index 2bfd50bd6a8..10f896037ab 100755
--- a/env/ORION.env
+++ b/env/ORION.env
@@ -311,9 +311,8 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
 elif [[ "${step}" = "epos" ]]; then
 
diff --git a/env/URSA.env b/env/URSA.env
index d4dc0161877..1b952df431c 100644
--- a/env/URSA.env
+++ b/env/URSA.env
@@ -291,9 +291,8 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
 elif [[ "${step}" = "epos" ]]; then
 
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index d9c6578a940..b0da0aa559a 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -321,10 +321,8 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks} "
-    #export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks}"
 
 elif [[ "${step}" = "epos" ]]; then
 

From 97c052daf33c6576cbe23782b960819278bafa17 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Fri, 31 Oct 2025 13:15:52 +0000
Subject: [PATCH 17/28] Debugging

---
 scripts/exgdas_enkf_sfc.sh       | 10 +++++-----
 ush/regrid_gsiSfcIncr_to_tile.sh |  6 +++---
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
index c22a2b6ee7c..5dff02f56b8 100755
--- a/scripts/exgdas_enkf_sfc.sh
+++ b/scripts/exgdas_enkf_sfc.sh
@@ -163,7 +163,7 @@ if [[ "$DOIAU" == "YES" ]]; then
 
         export TILE_NUM=$n
 
-	# Copy inputs from COMIN to DATA
+        # Copy inputs from COMIN to DATA
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
             if (( smem > NMEM_ENS_MAX )); then
@@ -207,13 +207,13 @@ if [[ "$DOIAU" == "YES" ]]; then
             fi
 
         done # ensembles
-         
-	"${CYCLESH}" && true
+
+        "${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
            err_exit "Failed to update surface fields!"
         fi
-        
+
         # Copy outputs from DATA to COMOUT
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
@@ -281,7 +281,7 @@ if [[ "${DOSFCANL_ENKF}" == "YES" ]]; then
                    "${DATA}/soil_xainc.${cmem}"
             fi
         done
-        
+
         "${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index e7afecead60..f6b14521ea6 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -50,7 +50,7 @@ done
 if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     IFS=',' read -ra landifhrs <<< "${IAUFHRS}"
 fi
-export in_fname="'enkfgdas.sfci'"
+export in_fname="'sfci'"
 export out_fname="'sfci'"
 export in_dir=""
 export dir_mask_in="'./'"
@@ -137,13 +137,13 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
 
     for FHR in "${soilinc_fhrs[@]}"; do
         echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \
-                    ${memdir}/enkfgdas.sfci00${FHR}.nc" >> "cmdfile.${imem}"
+                    ${memdir}/sfci00${FHR}.nc" >> "cmdfile.${imem}"
     done 
 
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
         for FHI in "${landifhrs[@]}"; do
             echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \
-                        ${memdir}/enkfgdas.sfci00${FHI}.nc" >> "cmdfile.${imem}"
+                        ${memdir}/sfci00${FHI}.nc" >> "cmdfile.${imem}"
         done
     fi
 

From 065eda90154a5a5ac50d2e12ac66432ff4a0b25f Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Fri, 31 Oct 2025 16:02:41 +0000
Subject: [PATCH 18/28] Debug

---
 scripts/exgdas_enkf_sfc.sh | 2 +-
 sorc/ufs_utils.fd          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
index 5dff02f56b8..df55c0d1aa6 100755
--- a/scripts/exgdas_enkf_sfc.sh
+++ b/scripts/exgdas_enkf_sfc.sh
@@ -201,7 +201,7 @@ if [[ "$DOIAU" == "YES" ]]; then
             cpreq "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc"     "${DATA}/fngrid.${cmem}"
             cpreq "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}"
 
-            if [[ "${DO_GSISOILDA}" == "YES" ]]; then
+            if [[ "${DO_GSISOILDA}" == "YES" ]] && [[ "${DO_LAND_IAU}" == ".false." ]]; then
                  cpreq "${COMIN_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${LFHR}.tile${n}.nc" \
                    "${DATA}/soil_xainc.${cmem}"
             fi
diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index f091f4ec024..d543abc5473 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit f091f4ec02459d9d20052264aa7cca31d633f259
+Subproject commit d543abc54733c3f3eec591668639bc0dd3b1dcb0

From 7b4aaafff6864daffa56dd4f54c940adde319498 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Mon, 3 Nov 2025 18:25:03 +0000
Subject: [PATCH 19/28] Update ufs_utils hash

---
 sorc/ufs_utils.fd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index d543abc5473..bbe8a251c78 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit d543abc54733c3f3eec591668639bc0dd3b1dcb0
+Subproject commit bbe8a251c7847d4a1a3e3205730f70fffb2671f3

From 6fde359991ff3ff10b2ac13387a51c5e64f47794 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 4 Nov 2025 13:50:59 +0000
Subject: [PATCH 20/28] Address reviewer comments

---
 dev/parm/config/gfs/config.resources | 3 +++
 ush/regrid_gsiSfcIncr_to_tile.sh     | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources
index b68ed76054a..1c47d8d2716 100644
--- a/dev/parm/config/gfs/config.resources
+++ b/dev/parm/config/gfs/config.resources
@@ -1319,6 +1319,9 @@ case ${step} in
     walltime="01:15:00"
     ntasks=$(( NMEM_ENS * 6))
     threads_per_task=1
+
+    # Use node_numerator/node_denominator of the tasks in a node
+    # to leave enough memory for regridding
     node_numerator=1
     node_denominator=2
     tasks_per_node=$(( node_numerator*max_tasks_per_node/node_denominator ))
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index f6b14521ea6..4c71a7f405c 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -81,7 +81,7 @@ rm -f cmdfile
 touch cmdfile
 chmod 755 cmdfile
 
-# Create MDMP command file for fixed files
+# Create MDMD command file for fixed files
 rm -f cmdfile.0
 touch cmdfile.0
 chmod 755 cmdfile.0

From 482503ebb68ea0c1cccec57618403ba9cade6b62 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Tue, 4 Nov 2025 15:25:52 +0000
Subject: [PATCH 21/28] Add clarifying comments

---
 sorc/ufs_utils.fd                |  2 +-
 ush/regrid_gsiSfcIncr_to_tile.sh | 22 +++++++++++++---------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index bbe8a251c78..908bbaaeb38 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit bbe8a251c7847d4a1a3e3205730f70fffb2671f3
+Subproject commit 908bbaaeb384045f168b04a51f0a008bf001f665
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 4c71a7f405c..7232958f4ff 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -109,10 +109,8 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
 
+    # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS
     if (( NMEM_REGRID > 1 )); then
-        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
-            COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
-
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
@@ -124,6 +122,7 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
         fi
         in_dir+="\"./${memchar}/\""        
     else
+        # If deterministic job, memdir is just DATA
         memdir="${DATA}"
 
         in_dir="'./'"
@@ -167,29 +166,35 @@ export out_dir="${in_dir}"
 
 if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
     for FHR in "${soilinc_fhrs[@]}"; do
+        # Set namelist variables
         export add_time_dim=".false."
         export time_list="${FHR}"
         export out_fname="'sfci00${FHR}'"
 
+        # Create regrid namelist
         rm -f "regrid.nml"
         atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
+        # Run regrid executable
         ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
     	export err=$?
-	if [[ ${err} -ne 0 ]]; then
-	    err_exit "${REGRID_EXEC} failed, ABORT!"
-	fi
+	    if [[ ${err} -ne 0 ]]; then
+	        err_exit "${REGRID_EXEC} failed, ABORT!"
+	    fi
     done
 fi
 
 if [[ "${DO_LAND_IAU}" = ".true." ]]; then
+    # Set namelist variables
     export add_time_dim=".true."
     export time_list="${IAUFHRS}"
     export out_fname="'sfci'"
 
+    # Create regrid namelist
     rm -f "regrid.nml"
     atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
+    # Run regrid executable
     export pgm="${REGRID_EXEC}"
 	${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
 	export err=$?
@@ -211,15 +216,14 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
 
+    # If deterministic job, COMOUT_ATMOS_ANALYSIS_MEM is just COMOUT_ATMOS_ANALYSIS
     if (( NMEM_REGRID > 1 )); then
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
-        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
-            COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
-
         memdir="${DATA}/${memchar}"
     else
+        # If deterministic job, memdir is just DATA
         memdir="${DATA}"
     fi
 

From ac9b70028e48b372d51c4838cd146977839bee82 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Wed, 5 Nov 2025 14:26:51 +0000
Subject: [PATCH 22/28] Update ufs_utils hash to develop and add some comments

---
 sorc/ufs_utils.fd                | 2 +-
 ush/regrid_gsiSfcIncr_to_tile.sh | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd
index 908bbaaeb38..154a3676735 160000
--- a/sorc/ufs_utils.fd
+++ b/sorc/ufs_utils.fd
@@ -1 +1 @@
-Subproject commit 908bbaaeb384045f168b04a51f0a008bf001f665
+Subproject commit 154a367673555910b57aeb5b064f2c45c2f61e05
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 7232958f4ff..b3f47a439c4 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -164,6 +164,8 @@ export out_dir="${in_dir}"
 # Regrid soil increments and save to COMOUT
 #
 
+# Increments for offline analysis
+# If land IAU --> deterministic only. If no land IAU --> both deterministic and ensemble
 if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
     for FHR in "${soilinc_fhrs[@]}"; do
         # Set namelist variables
@@ -184,6 +186,8 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]
     done
 fi
 
+# Increments for forecast job with land IAU
+# If land IAU --> deterministic and ensemble
 if [[ "${DO_LAND_IAU}" = ".true." ]]; then
     # Set namelist variables
     export add_time_dim=".true."

From 66174111a4c7d20a63faa13f56b9b856656cefe8 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Wed, 5 Nov 2025 14:49:17 +0000
Subject: [PATCH 23/28] UPdate

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index b3f47a439c4..63fc2d32811 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -6,6 +6,7 @@ source "${HOMEgfs}/ush/atparse.bash"
 # Script to regrid surface increment from GSI grid
 # to fv3 tiles.
 # Clara Draper, Dec 2024
+# David New, Nov 2025 (parallelization updates)
 #-------------------------------------------------------------------------------------------------
 
 export PGMOUT=${PGMOUT:-${pgmout:-'&1'}}

From f833970506ddc3eaada0e7c130925d9821ceda48 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 6 Nov 2025 18:38:53 +0000
Subject: [PATCH 24/28] Address reviewer comments

---
 jobs/JGDAS_ENKF_SFC              |  3 ++
 jobs/JGLOBAL_ATMOS_SFCANL        |  3 ++
 ush/regrid_gsiSfcIncr_to_tile.sh | 82 ++++++++++++++++++--------------
 3 files changed, 51 insertions(+), 37 deletions(-)

diff --git a/jobs/JGDAS_ENKF_SFC b/jobs/JGDAS_ENKF_SFC
index 8ff4982e353..ee2bb8112a9 100755
--- a/jobs/JGDAS_ENKF_SFC
+++ b/jobs/JGDAS_ENKF_SFC
@@ -36,6 +36,9 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COMIN_OBS_PREV:COM_OBS_TMPL \
     COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
 
+# Use CFP to stage and save files in parallel
+ export USE_CFP=YES
+
 ###############################################################
 # Run relevant script
 
diff --git a/jobs/JGLOBAL_ATMOS_SFCANL b/jobs/JGLOBAL_ATMOS_SFCANL
index 6223c4cacf3..41fdeec5969 100755
--- a/jobs/JGLOBAL_ATMOS_SFCANL
+++ b/jobs/JGLOBAL_ATMOS_SFCANL
@@ -32,6 +32,9 @@ RUN="enkfgdas" MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 
 mkdir -p "${COMOUT_ATMOS_RESTART}"
 
+# Use CFP to stage and save files in parallel
+ export USE_CFP=YES
+
 ###############################################################
 # Run relevant script
 
diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 63fc2d32811..22d8fbce29f 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -11,19 +11,10 @@ source "${HOMEgfs}/ush/atparse.bash"
 
 export PGMOUT=${PGMOUT:-${pgmout:-'&1'}}
 export PGMERR=${PGMERR:-${pgmerr:-'&2'}}
-export REDOUT=${REDOUT:-'1>'}
-export REDERR=${REDERR:-'2>'}
 
 export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
-# Use CFP for ensemble job
-if [[ "${RUN}" == *"enkf"* ]]; then
-    export USE_CFP=YES
-else
-    export USE_CFP=NO
-fi
-
 NMEM_REGRID=${NMEM_REGRID:-1}
 CASE_IN=${CASE_IN:-${CASE_ENS}}
 LFHR=${LFHR:-6}
@@ -77,34 +68,30 @@ fi
 # Stage input files
 #
 
-# Create master MPMD command file
-rm -f cmdfile
-touch cmdfile
-chmod 755 cmdfile
-
 # Create MDMD command file for fixed files
 rm -f cmdfile.0
 touch cmdfile.0
 chmod 755 cmdfile.0
+
+# Append fixed files command file to master command file
+{
 echo "#!/bin/bash" > cmdfile.0
 
 # input, fixed files
 echo "cpreq ${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc \
-            ${DATA}/gaussian_scrip.nc" >> cmdfile.0
+            ${DATA}/gaussian_scrip.nc"
 
 # output, fixed files
 echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc \
-            ${DATA}/${CASE_OUT}_mosaic.nc" >> cmdfile.0
+            ${DATA}/${CASE_OUT}_mosaic.nc"
 
 for n in $(seq 1 "${ntiles}"); do
     echo "cpreq ${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc \
-                ${DATA}/vegetation_type.tile${n}.nc" >> cmdfile.0
+                ${DATA}/vegetation_type.tile${n}.nc"
     echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc \
-                ${DATA}/${CASE_OUT}_grid.tile${n}.nc" >> cmdfile.0
+                ${DATA}/${CASE_OUT}_grid.tile${n}.nc"
 done
-
-# Append fixed files command file to master command file
-echo "${DATA}/cmdfile.0" >> cmdfile
+} > cmdfile.0
 
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
@@ -133,23 +120,37 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     rm -f "cmdfile.${imem}"
     touch "cmdfile.${imem}"
     chmod 755 "cmdfile.${imem}"
+
+    # Create commands to stage input files
+    {
     echo "#!/bin/bash" > "cmdfile.${imem}"
 
     for FHR in "${soilinc_fhrs[@]}"; do
         echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \
-                    ${memdir}/sfci00${FHR}.nc" >> "cmdfile.${imem}"
+                    ${memdir}/sfci00${FHR}.nc"
     done 
 
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
         for FHI in "${landifhrs[@]}"; do
             echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \
-                        ${memdir}/sfci00${FHI}.nc" >> "cmdfile.${imem}"
+                        ${memdir}/sfci00${FHI}.nc"
         done
     fi
+    } > "cmdfile.${imem}"
+done
 
-    # Append this member's command file to master command file
-    echo "${DATA}/cmdfile.${imem}" >> cmdfile
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 755 cmdfile
+
+# Append all members' command files to master command file
+{
+echo "${DATA}/cmdfile.0" # fixed files
+for imem in $(seq 1 "${NMEM_REGRID}"); do
+    echo "${DATA}/cmdfile.${imem}"
 done
+} >> cmdfile
 
 # Run MPMD to stage input files
 "${USHgfs}/run_mpmd.sh" "cmdfile" && true
@@ -179,7 +180,7 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]
         atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
         # Run regrid executable
-        ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+        ${APRUN_REGRID} "${REGRID_EXEC}" "1>${PGMOUT}" "2>${PGMERR}"
     	export err=$?
 	    if [[ ${err} -ne 0 ]]; then
 	        err_exit "${REGRID_EXEC} failed, ABORT!"
@@ -201,7 +202,7 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
 
     # Run regrid executable
     export pgm="${REGRID_EXEC}"
-	${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+	${APRUN_REGRID} "${REGRID_EXEC}" "1>${PGMOUT}" "2>${PGMERR}"
 	export err=$?
 	if [[ ${err} -ne 0 ]]; then
 	    err_exit "${pgm} failed, ABORT!"
@@ -212,11 +213,6 @@ fi
 # Save regridded files to COMOUT
 #
 
-# Create master MPMD command file
-rm -f cmdfile
-touch cmdfile
-chmod 755 cmdfile
-
 for imem in $(seq 1 "${NMEM_REGRID}"); do
     cmem=$(printf %03i "${imem}")
     memchar="mem${cmem}"
@@ -236,13 +232,15 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     rm -f "cmdfile.${imem}"
     touch "cmdfile.${imem}"
     chmod 755 "cmdfile.${imem}"
-    echo "#!/bin/bash" > "cmdfile.${imem}"
+
+    {
+    echo "#!/bin/bash"
 
     if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
         for FHR in "${soilinc_fhrs[@]}"; do
             for n in $(seq 1 "${ntiles}"); do
                 echo "cpfs ${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \
-                      ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" >> "cmdfile.${imem}"
+                      ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc"
             done
         done
     fi
@@ -250,13 +248,23 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
     if [[ "${DO_LAND_IAU}" = ".true." ]]; then
         for n in $(seq 1 "${ntiles}"); do
             echo "cpfs ${memdir}/sfci.mem${imem}.tile${n}.nc \
-                  ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i006.tile${n}.nc" >> "cmdfile.${imem}"
+                  ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i006.tile${n}.nc"
         done
     fi
+    } > "cmdfile.${imem}"
+done
+
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 755 cmdfile
 
-    # Append this member's command file to master command file
-    echo "${DATA}/cmdfile.${imem}" >> cmdfile
+# Append all members' command files to master command file
+{
+for imem in $(seq 1 "${NMEM_REGRID}"); do
+    echo "${DATA}/cmdfile.${imem}"
 done
+} >> cmdfile
 
 # Run MPMD to save output files
 "${USHgfs}/run_mpmd.sh" "cmdfile" && true

From b4287f6e23019363a7ab6dfac7faaf0c4a39338d Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 6 Nov 2025 18:42:24 +0000
Subject: [PATCH 25/28] Address another comment

---
 dev/parm/config/gfs/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources
index 1c47d8d2716..2f3829622eb 100644
--- a/dev/parm/config/gfs/config.resources
+++ b/dev/parm/config/gfs/config.resources
@@ -1317,7 +1317,7 @@ case ${step} in
 
   "esfc")
     walltime="01:15:00"
-    ntasks=$(( NMEM_ENS * 6))
+    ntasks=$(( NMEM_ENS * 6)) # The regrid program requires tasks to be tiles times members
     threads_per_task=1
 
     # Use node_numerator/node_denominator of the tasks in a node

From dbff6004a6d3fd901a92acb939bcd6cfba4ee01e Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 6 Nov 2025 19:10:03 +0000
Subject: [PATCH 26/28] Missed two carrots

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 22d8fbce29f..d6f3eff884a 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -75,7 +75,7 @@ chmod 755 cmdfile.0
 
 # Append fixed files command file to master command file
 {
-echo "#!/bin/bash" > cmdfile.0
+echo "#!/bin/bash"
 
 # input, fixed files
 echo "cpreq ${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc \
@@ -123,7 +123,7 @@ for imem in $(seq 1 "${NMEM_REGRID}"); do
 
     # Create commands to stage input files
     {
-    echo "#!/bin/bash" > "cmdfile.${imem}"
+    echo "#!/bin/bash"
 
     for FHR in "${soilinc_fhrs[@]}"; do
         echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \

From b74707eac414aa52456bc65a0e1dfecd1cf68329 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 6 Nov 2025 19:11:54 +0000
Subject: [PATCH 27/28] Take redirects out of quotes

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index d6f3eff884a..5321cda03e8 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -180,7 +180,7 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]
         atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
         # Run regrid executable
-        ${APRUN_REGRID} "${REGRID_EXEC}" "1>${PGMOUT}" "2>${PGMERR}"
+        ${APRUN_REGRID} "${REGRID_EXEC}" 1>${PGMOUT}" 2>${PGMERR}"
     	export err=$?
 	    if [[ ${err} -ne 0 ]]; then
 	        err_exit "${REGRID_EXEC} failed, ABORT!"
@@ -202,7 +202,7 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
 
     # Run regrid executable
     export pgm="${REGRID_EXEC}"
-	${APRUN_REGRID} "${REGRID_EXEC}" "1>${PGMOUT}" "2>${PGMERR}"
+	${APRUN_REGRID} "${REGRID_EXEC}" 1>${PGMOUT}" 2>${PGMERR}"
 	export err=$?
 	if [[ ${err} -ne 0 ]]; then
 	    err_exit "${pgm} failed, ABORT!"

From c6bb2ee97b53b7a0778754b5fd90aa3c977aa087 Mon Sep 17 00:00:00 2001
From: DavidNew-NOAA <david.new@noaa.gov>
Date: Thu, 6 Nov 2025 19:19:42 +0000
Subject: [PATCH 28/28] oops

---
 ush/regrid_gsiSfcIncr_to_tile.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh
index 5321cda03e8..30f9f6067dc 100755
--- a/ush/regrid_gsiSfcIncr_to_tile.sh
+++ b/ush/regrid_gsiSfcIncr_to_tile.sh
@@ -180,7 +180,7 @@ if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]
         atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
         # Run regrid executable
-        ${APRUN_REGRID} "${REGRID_EXEC}" 1>${PGMOUT}" 2>${PGMERR}"
+        ${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}"
     	export err=$?
 	    if [[ ${err} -ne 0 ]]; then
 	        err_exit "${REGRID_EXEC} failed, ABORT!"
@@ -202,7 +202,7 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
 
     # Run regrid executable
     export pgm="${REGRID_EXEC}"
-	${APRUN_REGRID} "${REGRID_EXEC}" 1>${PGMOUT}" 2>${PGMERR}"
+	${APRUN_REGRID} "${REGRID_EXEC}" 1>"${PGMOUT}" 2>"${PGMERR}"
 	export err=$?
 	if [[ ${err} -ne 0 ]]; then
 	    err_exit "${pgm} failed, ABORT!"