From b4e0619b82127f2e8bbef335684fb90c7662f637 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 31 Mar 2022 02:56:13 +0000 Subject: [PATCH 1/8] Modify RRFS North America 3- and 13-km domain configuration and WE2E test. --- ...s_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh | 4 +- ush/set_predef_grid_params.sh | 46 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index c088805de..4f95e4dd0 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -14,7 +14,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_NA_3km" +PREDEF_GRID_NAME="RRFS_NA_13km" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="FV3GFS" @@ -45,7 +45,7 @@ PPN_MAKE_ICS="4" PPN_MAKE_LBCS="4" WTIME_MAKE_LBCS="01:00:00" -NNODES_RUN_POST="6" +NNODES_RUN_POST="8" PPN_RUN_POST="12" OMP_STACKSIZE_MAKE_ICS="2048m" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index dd664670a..3d301dd42 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1167,14 +1167,14 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR="-106.0" - ESGgrid_LAT_CTR="54.0" + ESGgrid_LON_CTR="-112.5" + ESGgrid_LAT_CTR="55.0" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX="960" - ESGgrid_NY="960" + ESGgrid_NX="912" + ESGgrid_NY="623" ESGgrid_PAZI="0.0" @@ -1190,12 +1190,12 @@ case ${PREDEF_GRID_NAME} in WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group="16" WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-55.82538869" - WRTCMP_lat_lwr_left="-48.57685654" - WRTCMP_lon_upr_rght="55.82538869" - WRTCMP_lat_upr_rght="48.57685654" + WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" + WRTCMP_lon_lwr_left="-61.0" + WRTCMP_lat_lwr_left="-37.0" + WRTCMP_lon_upr_rght="61.0" + WRTCMP_lat_upr_rght="37.0" WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi @@ -1212,35 +1212,35 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-107.5 - ESGgrid_LAT_CTR=51.5 + ESGgrid_LON_CTR=-112.5 + ESGgrid_LAT_CTR=55.0 ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=3640 - ESGgrid_NY=2520 + ESGgrid_NX=3950 + ESGgrid_NY=2700 - ESGgrid_PAZI="-13.0" + ESGgrid_PAZI="0.0" ESGgrid_WIDE_HALO_WIDTH=6 DT_ATMOS="${DT_ATMOS:-36}" - LAYOUT_X="${LAYOUT_X:-18}" # 40 - EMC operational configuration - LAYOUT_Y="${LAYOUT_Y:-36}" # 45 - EMC operational configuration + LAYOUT_X="${LAYOUT_X:-20}" # 40 - EMC operational configuration + LAYOUT_Y="${LAYOUT_Y:-35}" # 45 - EMC operational configuration BLOCKSIZE="${BLOCKSIZE:-28}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group="144" WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-51.0" - WRTCMP_lat_lwr_left="-33.0" - WRTCMP_lon_upr_rght="51.0" - WRTCMP_lat_upr_rght="33.0" + WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" + WRTCMP_lon_lwr_left="-61.0" + WRTCMP_lat_lwr_left="-37.0" + WRTCMP_lon_upr_rght="61.0" + WRTCMP_lat_upr_rght="37.0" WRTCMP_dlon="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi From 6f88d016f08fe6e6316f919f7ad6aca13222a1a6 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 31 Mar 2022 02:57:45 +0000 Subject: [PATCH 2/8] Change sotyp_from_climo to "true" based on operational RAP grib2 files. --- scripts/exregional_make_ics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 8a63b1978..8a4b83445 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -437,7 +437,7 @@ case "${EXTRN_MDL_NAME_ICS}" in # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_RAPX" vgtyp_from_climo=True - sotyp_from_climo=False + sotyp_from_climo=True vgfrc_from_climo=True minmax_vgfrc_from_climo=True lai_from_climo=True From 035e7c841af397b817ebe2c3ea0cd4c23d78d3a6 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 31 Mar 2022 02:59:10 +0000 Subject: [PATCH 3/8] Update for changes to stochastic physics namelist options. --- ush/generate_FV3LAM_wflow.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index d0f5d3bd4..83282b717 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -856,8 +856,9 @@ settings="$settings fi # # Add the relevant tendency-based stochastic physics namelist variables to -# "settings" when running with SPPT, SHUM, or SKEB turned on. Otherwise -# only include an empty "nam_stochy" stanza. +# "settings" when running with SPPT, SHUM, or SKEB turned on. If running +# with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only +# include an empty "nam_stochy" stanza. # settings="$settings 'nam_stochy': {" @@ -881,8 +882,7 @@ if [ "${DO_SHUM}" = "TRUE" ]; then 'shum': ${SHUM_MAG}, 'shum_lscale': ${SHUM_LSCALE}, 'shum_tau': ${SHUM_TSCALE}, - 'shumint': ${SHUM_INT}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," + 'shumint': ${SHUM_INT}," fi if [ "${DO_SKEB}" = "TRUE" ]; then @@ -894,8 +894,12 @@ if [ "${DO_SKEB}" = "TRUE" ]; then 'skebnorm': ${SKEBNORM}, 'skeb_tau': ${SKEB_TSCALE}, 'skebint': ${SKEB_INT}, - 'skeb_vdof': ${SKEB_VDOF}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," + 'skeb_vdof': ${SKEB_VDOF}," +fi + +if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then + settings="$settings + 'new_lscale': ${NEW_LSCALE}," fi settings="$settings }" From fa4317ffeb84e9b349c99835e3810048179efd61 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 31 Mar 2022 02:59:50 +0000 Subject: [PATCH 4/8] Check for DO_ENSEMBLE="TRUE" when running ensemble verification and turn of VX when running in NCO mode. --- ush/setup.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index 01470963d..b9711d588 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1393,6 +1393,21 @@ fi # #----------------------------------------------------------------------- # +# Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. +# +#----------------------------------------------------------------------- +# +if [ "${DO_ENSEMBLE}" = "FALSE" ] && [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" -o \ + "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then + print_err_msg_exit "\ +Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"${DO_ENSEMBLE}\" + RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" + RUN_TASK_VX_ENSPOINT = \"${RUN_TASK_VX_ENSPOINT}\"" +fi +# +#----------------------------------------------------------------------- +# # Set the full path to the forecast model executable. # #----------------------------------------------------------------------- @@ -1576,8 +1591,7 @@ one above. Reset values are: fi - if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -1596,8 +1610,7 @@ Reset value is:" fi - if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -1616,8 +1629,7 @@ Reset value is:" fi - if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -2011,7 +2023,6 @@ SUB_HOURLY_POST is NOT available with Inline Post yet." fi fi - check_var_valid_value "QUILTING" "valid_vals_QUILTING" QUILTING=$(boolify $QUILTING) From cc0e8a4e4b8858f6855d09b8530291de7ea09826 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 31 Mar 2022 03:31:54 +0000 Subject: [PATCH 5/8] Revert to 3-km domain. --- ...rid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index 4f95e4dd0..f743ebb40 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -14,7 +14,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_NA_13km" +PREDEF_GRID_NAME="RRFS_NA_3km" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="FV3GFS" From ace5ef828dfdc0ba79e44c1abc11f7a1bca59a75 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Fri, 1 Apr 2022 16:29:13 +0000 Subject: [PATCH 6/8] Remove commented-out GFDL grid for the RRFS_NA_13km domain --- ush/set_predef_grid_params.sh | 44 ----------------------------------- 1 file changed, 44 deletions(-) diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 3d301dd42..808dd4c91 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1121,50 +1121,6 @@ case ${PREDEF_GRID_NAME} in # "RRFS_NA_13km") -# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then -# -# GFDLgrid_LON_T6_CTR="-106.0" -# GFDLgrid_LAT_T6_CTR="54.0" -# GFDLgrid_STRETCH_FAC="0.63" -# GFDLgrid_RES="384" -# GFDLgrid_REFINE_RATIO="3" -# -# num_margin_cells_T6_left="10" -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) -# -# num_margin_cells_T6_right="10" -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) -# -# num_margin_cells_T6_bottom="10" -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) -# -# num_margin_cells_T6_top="10" -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) -# -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" -# -# DT_ATMOS="50" -# -# LAYOUT_X="14" -# LAYOUT_Y="14" -# BLOCKSIZE="26" -# -# if [ "$QUILTING" = "TRUE" ]; then -# WRTCMP_write_groups="1" -# WRTCMP_write_tasks_per_group="14" -# WRTCMP_output_grid="rotated_latlon" -# WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}" -# WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}" -# WRTCMP_lon_lwr_left="-57.9926" -# WRTCMP_lat_lwr_left="-50.74344" -# WRTCMP_lon_upr_rght="57.99249" -# WRTCMP_lat_upr_rght="50.74344" -# WRTCMP_dlon="0.1218331" -# WRTCMP_dlat="0.121833" -# fi -# -# elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR="-112.5" From 1c52bbd1f80ac969f66abd03bd40b5c82e3bff33 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Fri, 1 Apr 2022 16:53:51 +0000 Subject: [PATCH 7/8] Add RRFS_NA_13km WE2E test --- ...cs_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh new file mode 100644 index 000000000..3cad44e2b --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -0,0 +1,54 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_NA_3km grid using the RRFS_v1alpha +# physics suite with ICs and LBCs derived from the FV3GFS. +# +# Note that this test also sets various resource parameters for several +# of the rocoto tasks in order to more efficiently run the code on this +# (very large) grid. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_NA_13km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +######################################################################### +# The following code/namelist/workflow setting changes are necessary to # +# run/optimize end-to-end experiments using the 3-km NA grid # +######################################################################### + +# The model should be built in 32-bit mode (64-bit will result in much +# longer run times. + +# Use k_split=2 and n_split=5, the previous namelist values (k_split=4 +# and n_split=5) will result in significantly longer run times. + +NNODES_MAKE_ICS="12" +NNODES_MAKE_LBCS="12" +PPN_MAKE_ICS="4" +PPN_MAKE_LBCS="4" +WTIME_MAKE_LBCS="01:00:00" + +NNODES_RUN_POST="6" +PPN_RUN_POST="12" + +OMP_STACKSIZE_MAKE_ICS="2048m" +OMP_STACKSIZE_RUN_FCST="2048m" + +############################################################################### From 9ca8bd93043d8ab6cea4c298738ec68aa84f79e9 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Fri, 1 Apr 2022 18:37:41 +0000 Subject: [PATCH 8/8] Changes to comments. --- ...rid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh index 3cad44e2b..9e4cb594b 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -3,7 +3,7 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_NA_3km grid using the RRFS_v1alpha +# completes successfully on the RRFS_NA_13km grid using the RRFS_v1beta # physics suite with ICs and LBCs derived from the FV3GFS. # # Note that this test also sets various resource parameters for several