Add the global nesting capability

parm/config/gefs/config.base

@@ -137,6 +137,13 @@ export DO_AERO="NO"
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both
 export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points
 export FRAC_GRID=".true."
+export DO_NEST="NO" # Whether to run a global-nested domain
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export ntiles=7
+  export NEST_OUTPUT_GRID="regional_latlon"
+else
+  export ntiles=6
+fi
 
 # Set operational resolution
 export OPS_RES="C768" # Do not change

parm/config/gfs/config.base

@@ -184,6 +184,13 @@ export DO_AERO="NO"
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both
 export DOBNDPNT_WAVE="NO"
 export FRAC_GRID=".true."
+export DO_NEST="NO" # Whether to run a global-nested domain
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export ntiles=7
+  export NEST_OUTPUT_GRID="regional_latlon"
+else
+  export ntiles=6
+fi
 
 # Set operational resolution
 export OPS_RES="C768" # Do not change # TODO: Why is this needed and where is it used?

parm/config/gfs/config.fcst

@@ -254,6 +254,9 @@ export DO_SKEB=${DO_SKEB:-"NO"}
 export DO_SHUM=${DO_SHUM:-"NO"}
 export DO_LAND_PERT=${DO_LAND_PERT:-"NO"}
 export DO_CA=${DO_CA:-"YES"}
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export DO_CA="NO"  # CA does not work with nesting.
+fi
 
 #coupling settings
 export cplmode="ufs.frac"

parm/config/gfs/config.resources

@@ -606,6 +606,13 @@ case ${step} in
         [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
       fi
 
+      if (( ntiles > 6 )); then
+        export layout_x_nest=${layout_x_nest:-10}
+        export layout_y_nest=${layout_y_nest:-10}
+        export npx_nest=${npx_nest:-1441}
+        export npy_nest=${npy_nest:-961}
+      fi
+
       # PETS for the atmosphere dycore
       (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
       echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"

parm/config/gfs/config.stage_ic

@@ -38,4 +38,8 @@ case "${CASE}" in
     ;;
 esac
 
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export CPL_ATMIC="GLOBAL-NEST_${CASE}"
+fi
+
 echo "END: config.stage_ic"

parm/config/gfs/config.ufs

@@ -68,6 +68,22 @@ if [[ "${skip_mom6}" == "false" ]] || [[ "${skip_cice6}" == "false" ]] || [[ "${
   skip_mediator=false
 fi
 
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  # Describe nest location, interaction with parent, etc.
+  export grid_type=0
+  export stretch_fac=1.0001
+  export TARGET_LAT=32.5
+  export TARGET_LON=-135.0
+  export NEST_LON1=-195.000000
+  export NEST_LAT1=-7.500000
+  export NEST_LON2=-75.000000
+  export NEST_LAT2=72.500000
+  export twowaynest=${twowaynest:-.true.}
+else
+  # No nest.
+  export grid_type=-1
+fi
+
 # (Standard) Model resolution dependent variables
 case "${fv3_res}" in
     "C48")
@@ -89,76 +105,158 @@ case "${fv3_res}" in
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
         ;;
     "C96")
-        export DELTIM=600
-        export layout_x=2
-        export layout_y=2
-        export layout_x_gfs=2
-        export layout_y_gfs=2
-        export nthreads_fv3=1
-        export nthreads_fv3_gfs=1
-        export nthreads_ufs=1
-        export nthreads_ufs_gfs=1
-        export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
-        export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
-        export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
-        export WRITE_GROUP=1
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
-        export WRITE_GROUP_GFS=1
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
+        if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+          export DELTIM=450
+          export layout_x_gfs=4
+          export layout_y_gfs=4
+          export layout_x_nest=12
+          export layout_y_nest=10
+          export nest_refine=4
+          export nest_ioffset=4
+          export nest_joffset=9
+          export npx_nest=361
+          export npy_nest=241
+          export NEST_DLON=0.25
+          export NEST_DLAT=0.25
+          export WRITE_GROUP=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=2
+          export WRITE_GROUP_GFS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=2
+        else
+          export DELTIM=600
+          export layout_x=2
+          export layout_y=2
+          export layout_x_gfs=2
+          export layout_y_gfs=2
+          export nthreads_fv3=1
+          export nthreads_fv3_gfs=1
+          export nthreads_ufs=1
+          export nthreads_ufs_gfs=1
+          export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+          export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
+          export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
+          export WRITE_GROUP=1
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
+          export WRITE_GROUP_GFS=1
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
+        fi
         ;;
     "C192")
-        export DELTIM=450
-        export layout_x=4
-        export layout_y=6
-        export layout_x_gfs=4
-        export layout_y_gfs=6
-        export nthreads_fv3=1
-        export nthreads_fv3_gfs=2
-        export nthreads_ufs=1
-        export nthreads_ufs_gfs=2
-        export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
-        export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
-        export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
-        export WRITE_GROUP=1
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
-        export WRITE_GROUP_GFS=2
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=5
+        if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+          export DELTIM=225
+          export layout_x_gfs=5
+          export layout_y_gfs=6
+          export layout_x_nest=15
+          export layout_y_nest=25
+          export nest_refine=4
+          export nest_ioffset=7
+          export nest_joffset=19
+          export npx_nest=721
+          export npy_nest=481
+          export NEST_DLON=0.125
+          export NEST_DLAT=0.125
+          export WRITE_GROUP=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=15
+          export WRITE_GROUP_GFS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=15
+        else
+          export DELTIM=450
+          export layout_x=4
+          export layout_y=6
+          export layout_x_gfs=4
+          export layout_y_gfs=6
+          export nthreads_fv3=1
+          export nthreads_fv3_gfs=2
+          export nthreads_ufs=1
+          export nthreads_ufs_gfs=2
+          export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+          export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
+          export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
+          export WRITE_GROUP=1
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GFS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=5
+        fi
         ;;
     "C384")
-        export DELTIM=300
-        export layout_x=8
-        export layout_y=8
-        export layout_x_gfs=8
-        export layout_y_gfs=8
-        export nthreads_fv3=2
-        export nthreads_fv3_gfs=2
-        export nthreads_ufs=2
-        export nthreads_ufs_gfs=2
-        export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
-        export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
-        export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
-        export WRITE_GROUP=4
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
-        export WRITE_GROUP_GFS=4
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
+        if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+          export DELTIM=150
+          export layout_x=8
+          export layout_y=8
+          export layout_x_gfs=8
+          export layout_y_gfs=8
+          export layout_x_nest=34
+          export layout_y_nest=24
+          export nest_refine=4
+          export nest_ioffset=13
+          export nest_joffset=37
+          export npx_nest=1441
+          export npy_nest=961
+          export NEST_DLON=0.0625
+          export NEST_DLAT=0.0625
+          export WRITE_GROUP=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20
+          export WRITE_GROUP_GFS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20
+        else
+          export DELTIM=300
+          export layout_x=8
+          export layout_y=8
+          export layout_x_gfs=8
+          export layout_y_gfs=8
+          export nthreads_fv3=2
+          export nthreads_fv3_gfs=2
+          export nthreads_ufs=2
+          export nthreads_ufs_gfs=2
+          export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+          export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
+          export knob_ugwp_tauamp=0.8e-3      # setting for UGWPv1 non-stationary GWD
+          export WRITE_GROUP=4
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GFS=4
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
+        fi
         ;;
     "C768")
-        export DELTIM=150
-        export layout_x=8
-        export layout_y=12
-        export layout_x_gfs=12
-        export layout_y_gfs=16
-        export nthreads_fv3=4
-        export nthreads_fv3_gfs=4
-        export nthreads_ufs=4
-        export nthreads_ufs_gfs=4
-        export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
-        export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
-        export knob_ugwp_tauamp=0.5e-3      # setting for UGWPv1 non-stationary GWD
-        export WRITE_GROUP=2
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
-        export WRITE_GROUP_GFS=4
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2
+        if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+          export DELTIM=75
+          export layout_x=16
+          export layout_y=10
+          export layout_x_gfs=16
+          export layout_y_gfs=10
+          export layout_x_nest=48
+          export layout_y_nest=45
+          export nthreads_fv3=2
+          export nthreads_fv3_gfs=2
+          export nest_refine=4
+          export nest_ioffset=24
+          export nest_joffset=72
+          export npx_nest=2881
+          export npy_nest=1921
+          export NEST_DLON=0.0325
+          export NEST_DLAT=0.0325
+          export WRITE_GROUP=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=90
+          export WRITE_GROUP_GFS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=90
+        else
+          export DELTIM=150
+          export layout_x=8
+          export layout_y=12
+          export layout_x_gfs=12
+          export layout_y_gfs=16
+          export nthreads_fv3=4
+          export nthreads_fv3_gfs=4
+          export nthreads_ufs=4
+          export nthreads_ufs_gfs=4
+          export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+          export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
+          export knob_ugwp_tauamp=0.5e-3      # setting for UGWPv1 non-stationary GWD
+          export WRITE_GROUP=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GFS=4
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2
+        fi
         ;;
     "C1152")
         export DELTIM=120
@@ -209,6 +307,9 @@ export WRTTASK_PER_GROUP_PER_THREAD_GFS
 
 (( ntasks_fv3 = layout_x * layout_y * 6 ))
 (( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 ))
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  (( ntasks_fv3_gfs += layout_x_nest * layout_y_nest ))
+fi
 export ntasks_fv3
 export ntasks_fv3_gfs
 
@@ -241,7 +342,11 @@ export cplice=".false."
 export cplchm=".false."
 export cplwav=".false."
 export cplwav2atm=".false."
-export CCPP_SUITE="${CCPP_SUITE:-FV3_GFS_v17_p8_ugwpv1}"
+if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
+  export CCPP_SUITE="${CCPP_SUITE:-FV3_global_nest_v1}"
+else
+  export CCPP_SUITE="${CCPP_SUITE:-FV3_GFS_v17_p8_ugwpv1}"
+fi
 model_list="atm"
 
 # Mediator specific settings

scripts/exglobal_stage_ic.sh

@@ -42,9 +42,13 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
       err=$((err + rc))
     done
     for ftype in ca_data fv_core.res fv_srf_wnd.res fv_tracer.res phy_data sfc_data; do
-      for ((tt = 1; tt <= 6; tt++)); do
+      for ((tt = 1; tt <= ntiles; tt++)); do
         src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
-        tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+        if (( tt > 6 )) ; then
+            tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.nest0$((tt-5)).tile${tt}.nc"
+        else
+            tgt="${COM_ATMOS_RESTART_PREV}/${PDY}.${cyc}0000.${ftype}.tile${tt}.nc"
+        fi
         ${NCP} "${src}" "${tgt}"
         rc=$?
         ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
@@ -62,14 +66,17 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
     ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
     err=$((err + rc))
     for ftype in gfs_data sfc_data; do
-      for ((tt = 1; tt <= 6; tt++)); do
+      for ((tt = 1; tt <= ntiles; tt++)); do
         src="${BASE_CPLIC}/${CPL_ATMIC:-}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc"
         tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
         ${NCP} "${src}" "${tgt}"
         rc=$?
         ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
         err=$((err + rc))
       done
+      if (( ntiles > 6 )); then
+        ${NLN} "${COM_ATMOS_INPUT}/${ftype}.tile7.nc" "${COM_ATMOS_INPUT}/${ftype}.nest02.tile7.nc"
+      fi
     done
   fi
 

sorc/link_workflow.sh

@@ -178,7 +178,7 @@ done
 
 # Link these templates from ufs-weather-model
 cd "${HOMEgfs}/parm/ufs" || exit 1
-declare -a ufs_templates=("model_configure.IN" \
+declare -a ufs_templates=("model_configure.IN" "model_configure_nest.IN"\
                           "MOM_input_025.IN" "MOM_input_050.IN" "MOM_input_100.IN" "MOM_input_500.IN" \
                           "MOM6_data_table.IN" \
                           "ice_in.IN" \