Parallelize surface increment regridding

dev/parm/config/gfs/config.resources

@@ -1317,9 +1317,12 @@ case ${step} in
 
   "esfc")
     walltime="01:15:00"
-    ntasks=80
+    ntasks=$(( NMEM_ENS * 6))
     threads_per_task=1
-    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    node_numerator=1
+    node_denominator=2
+    tasks_per_node=$(( node_numerator*max_tasks_per_node/node_denominator ))
+
     threads_per_task_cycle=${threads_per_task}
     tasks_per_node_cycle=$(( max_tasks_per_node / threads_per_task_cycle ))
     ;;

env/HERA.env

@@ -302,9 +302,8 @@ elif [[ "${step}" = "esfc" ]]; then
     fi
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
-    # REGRID requires 6 tasks for reproducibility
-    ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    # REGRID requires 6*NMEM_ENS tasks for reproducibility
+    export APRUN_REGRID="${launcher} -n ${ntasks} "
 
 elif [[ "${step}" = "epos" ]]; then
 

env/WCOSS2.env

@@ -323,7 +323,8 @@ elif [[ "${step}" = "esfc" ]]; then
 
     # REGRID requires 6 tasks for reproducibility
     ntasks_regrid=6
-    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+    export APRUN_REGRID="${launcher} -n ${ntasks} "
+    #export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
 
 elif [[ "${step}" = "epos" ]]; then
 

scripts/exgdas_enkf_sfc.sh

@@ -163,7 +163,7 @@ if [[ "$DOIAU" == "YES" ]]; then
 
         export TILE_NUM=$n
 
-        # Copy inputs from COMIN to DATA
+	# Copy inputs from COMIN to DATA
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
             if (( smem > NMEM_ENS_MAX )); then
@@ -207,13 +207,13 @@ if [[ "$DOIAU" == "YES" ]]; then
             fi
 
         done # ensembles
-
-        "${CYCLESH}" && true
+         
+	"${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
            err_exit "Failed to update surface fields!"
         fi
-
+        
         # Copy outputs from DATA to COMOUT
         for imem in $(seq 1 $NMEM_ENS); do
             smem=$((imem + mem_offset))
@@ -281,7 +281,7 @@ if [[ "${DOSFCANL_ENKF}" == "YES" ]]; then
                    "${DATA}/soil_xainc.${cmem}"
             fi
         done
-
+        
         "${CYCLESH}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then

ush/regrid_gsiSfcIncr_to_tile.sh

@@ -16,6 +16,13 @@ export REDERR=${REDERR:-'2>'}
 export PGM=${REGRID_EXEC}
 export pgm=${PGM}
 
+# Use CFP for ensemble job
+if [[ "${RUN}" == *"enkf"* ]]; then
+    export USE_CFP=YES
+else
+    export USE_CFP=NO
+fi
+
 NMEM_REGRID=${NMEM_REGRID:-1}
 CASE_IN=${CASE_IN:-${CASE_ENS}}
 LFHR=${LFHR:-6}
@@ -45,6 +52,7 @@ if [[ "${DO_LAND_IAU}" = ".true." ]]; then
 fi
 export in_fname="'enkfgdas.sfci'"
 export out_fname="'sfci'"
+export in_dir=""
 export dir_mask_in="'./'"
 export fname_mask_in="'NULL'"
 export ires=${LONB_CASE_IN}
@@ -54,21 +62,6 @@ export jreso=${CASE_OUT:1}
 
 regrid_nml_tmpl="${PARMgfs}/regrid_sfc/regrid.nml_tmpl" 
 
-# input, fixed files
-cpreq "${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc" \
-      "${DATA}/gaussian_scrip.nc"
-
-# output, fixed files
-cpreq "${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc" \
-      "${DATA}/${CASE_OUT}_mosaic.nc"
-
-for n in $(seq 1 "${ntiles}"); do
-    cpreq "${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc" \
-          "${DATA}/vegetation_type.tile${n}.nc"
-    cpreq "${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc" \
-          "${DATA}/${CASE_OUT}_grid.tile${n}.nc"
-done
-
 if (( LFHR >= 0 )); then
     soilinc_fhrs=("${LFHR}")
 else # construct restart times for deterministic member
@@ -79,63 +72,188 @@ else # construct restart times for deterministic member
     fi
 fi
 
+#
+# Stage input files
+#
+
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 755 cmdfile
+
+# Create MDMP command file for fixed files
+rm -f cmdfile.0
+touch cmdfile.0
+chmod 755 cmdfile.0
+echo "#!/bin/bash" > cmdfile.0
+
+# input, fixed files
+echo "cpreq ${FIXorog}/${CASE_IN}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc \
+            ${DATA}/gaussian_scrip.nc" >> cmdfile.0
+
+# output, fixed files
+echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc \
+            ${DATA}/${CASE_OUT}_mosaic.nc" >> cmdfile.0
+
+for n in $(seq 1 "${ntiles}"); do
+    echo "cpreq ${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc \
+                ${DATA}/vegetation_type.tile${n}.nc" >> cmdfile.0
+    echo "cpreq ${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc \
+                ${DATA}/${CASE_OUT}_grid.tile${n}.nc" >> cmdfile.0
+done
+
+# Append fixed files command file to master command file
+echo "${DATA}/cmdfile.0" >> cmdfile
+
 for imem in $(seq 1 "${NMEM_REGRID}"); do
-    if (( NMEM_REGRID > 1 )); then
-        cmem=$(printf %03i "${imem}")
-        memchar="mem${cmem}"
+    cmem=$(printf %03i "${imem}")
+    memchar="mem${cmem}"
 
+    if (( NMEM_REGRID > 1 )); then
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
 
         MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
             COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        memdir="${DATA}/${memchar}"
+        mkdir -p "${memdir}"
+
+        if [[ "${imem}" -gt 1 ]]; then
+            in_dir+=", "
+        fi
+        in_dir+="\"./${memchar}/\""        
+    else
+        memdir="${DATA}"
+
+        in_dir="'./'"
     fi
 
+    # Create MPMD command file for this member
+    rm -f "cmdfile.${imem}"
+    touch "cmdfile.${imem}"
+    chmod 755 "cmdfile.${imem}"
+    echo "#!/bin/bash" > "cmdfile.${imem}"
+
     for FHR in "${soilinc_fhrs[@]}"; do
+        echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc \
+                    ${memdir}/enkfgdas.sfci00${FHR}.nc" >> "cmdfile.${imem}"
+    done 
+
+    if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
+        for FHI in "${landifhrs[@]}"; do
+            echo "cpreq ${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc \
+                        ${memdir}/enkfgdas.sfci00${FHI}.nc" >> "cmdfile.${imem}"
+        done
+    fi
 
+    # Append this member's command file to master command file
+    echo "${DATA}/cmdfile.${imem}" >> cmdfile
+done
+
+# Run MPMD to stage input files
+"${USHgfs}/run_mpmd.sh" "cmdfile" && true
+export err=$?
+if [[ ${err} -ne 0 ]]; then
+    err_exit "run_mpmd.sh failed!"
+fi
+
+# Finish defining input/output directory list
+export out_dir="${in_dir}"
+
+#
+# Regrid soil increments and save to COMOUT
+#
+
+if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
+    for FHR in "${soilinc_fhrs[@]}"; do
         export add_time_dim=".false."
         export time_list="${FHR}"
+        export out_fname="'sfci00${FHR}'"
 
         rm -f "regrid.nml"
         atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
 
-        cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHR}.nc" \
-               "${DATA}/enkfgdas.sfci00${FHR}.nc"
-
         ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+    	export err=$?
+	if [[ ${err} -ne 0 ]]; then
+	    err_exit "${REGRID_EXEC} failed, ABORT!"
+	fi
+    done
+fi
 
-        for n in $(seq 1 "${ntiles}"); do
-            cpfs "${DATA}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc"
-        done
-    done 
+if [[ "${DO_LAND_IAU}" = ".true." ]]; then
+    export add_time_dim=".true."
+    export time_list="${IAUFHRS}"
+    export out_fname="'sfci'"
+
+    rm -f "regrid.nml"
+    atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+
+    export pgm="${REGRID_EXEC}"
+	${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
+	export err=$?
+	if [[ ${err} -ne 0 ]]; then
+	    err_exit "${pgm} failed, ABORT!"
+	fi
+fi
 
-    if [[ "${DO_LAND_IAU}" = ".true." ]]; then 
+#
+# Save regridded files to COMOUT
+#
 
-        export add_time_dim=".true."
-        export time_list="${IAUFHRS}"
+# Create master MPMD command file
+rm -f cmdfile
+touch cmdfile
+chmod 755 cmdfile
 
-        rm -f "regrid.nml"
-        atparse < "${regrid_nml_tmpl}" >> "regrid.nml"
+for imem in $(seq 1 "${NMEM_REGRID}"); do
+    cmem=$(printf %03i "${imem}")
+    memchar="mem${cmem}"
 
-        for FHI in "${landifhrs[@]}"; do
-            cpreq "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}increment.sfc.i00${FHI}.nc" \
-                  "${DATA}/enkfgdas.sfci00${FHI}.nc"
+    if (( NMEM_REGRID > 1 )); then
+        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+            COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+            COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL
+
+        memdir="${DATA}/${memchar}"
+    else
+        memdir="${DATA}"
+    fi
+
+    # Create MPMD command file for this member
+    rm -f "cmdfile.${imem}"
+    touch "cmdfile.${imem}"
+    chmod 755 "cmdfile.${imem}"
+    echo "#!/bin/bash" > "cmdfile.${imem}"
+
+    if [[ "${DO_LAND_IAU}" = ".false." || "${RUN}" == "gdas" || "${RUN}" == "gfs" ]]; then
+        for FHR in "${soilinc_fhrs[@]}"; do
+            for n in $(seq 1 "${ntiles}"); do
+                echo "cpfs ${memdir}/sfci00${FHR}.mem${imem}.tile${n}.nc \
+                      ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i00${FHR}.tile${n}.nc" >> "cmdfile.${imem}"
+            done
         done
-
-        export pgm="${REGRID_EXEC}"
-	      ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}"
-	      export err=$?
-	      if [[ ${err} -ne 0 ]]; then
-	          err_exit "${pgm} failed, ABORT!"
-	      fi
+    fi
 
+    if [[ "${DO_LAND_IAU}" = ".true." ]]; then
         for n in $(seq 1 "${ntiles}"); do
-            cpfs "${DATA}/sfci.tile${n}.nc"  "${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i006.tile${n}.nc"
+            echo "cpfs ${memdir}/sfci.mem${imem}.tile${n}.nc \
+                  ${COMOUT_ATMOS_ANALYSIS_MEM}/increment.sfc.i006.tile${n}.nc" >> "cmdfile.${imem}"
         done
-
     fi
 
+    # Append this member's command file to master command file
+    echo "${DATA}/cmdfile.${imem}" >> cmdfile
 done
 
-exit 0
+# Run MPMD to save output files
+"${USHgfs}/run_mpmd.sh" "cmdfile" && true
+export err=$?
+if [[ ${err} -ne 0 ]]; then
+    err_exit "run_mpmd.sh failed!"
+fi
 
+exit 0