From 3f9775700aadf0a51402e6f84acdd9b2ced3bd7e Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 22 Apr 2021 00:11:41 +0000 Subject: [PATCH 01/12] Add EMC's RRFS_NA_3km domain --- ush/set_predef_grid_params.sh | 41 +++++++++++++++++++++++++++++++++++ ush/valid_param_vals.sh | 1 + 2 files changed, 42 insertions(+) diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index cfb4136ba..853a64396 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1165,6 +1165,47 @@ case ${PREDEF_GRID_NAME} in WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi +# +#----------------------------------------------------------------------- +# +# Future operational RRFS domain with ~3km cell size. +# +#----------------------------------------------------------------------- +# +"RRFS_NA_3km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR=-107.5 + ESGgrid_LAT_CTR=51.5 + + ESGgrid_DELX="3000.0" + ESGgrid_DELY="3000.0" + + ESGgrid_NX=3640 + ESGgrid_NY=2520 + + ESGgrid_WIDE_HALO_WIDTH=6 + + DT_ATMOS="${DT_ATMOS:-36}" + + LAYOUT_X="${LAYOUT_X:-40}" + LAYOUT_Y="${LAYOUT_Y:-45}" + BLOCKSIZE="${BLOCKSIZE:-28}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="2" + WRTCMP_write_tasks_per_group="90" + WRTCMP_output_grid="rotated_latlon" + WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}" + WRTCMP_lon_lwr_left="-51.0" + WRTCMP_lat_lwr_left="-33.0" + WRTCMP_lon_upr_rght="51.0" + WRTCMP_lat_upr_rght="33.0" + WRTCMP_dlon="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) + WRTCMP_dlat="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) + fi ;; esac diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index b27ecafa6..f81948e96 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -20,6 +20,7 @@ valid_vals_PREDEF_GRID_NAME=( \ "GSL_HAFSV0.A_3km" \ "GSD_HRRR_AK_50km" \ "RRFS_NA_13km" \ +"RRFS_NA_3km" \ ) valid_vals_CCPP_PHYS_SUITE=( \ "FV3_CPT_v0" \ From 2b1299f265d86afccca0267b4ddda6182e09afa7 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Thu, 22 Apr 2021 00:16:25 +0000 Subject: [PATCH 02/12] Typo in case statement --- ush/set_predef_grid_params.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 853a64396..1670f2a83 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1165,6 +1165,7 @@ case ${PREDEF_GRID_NAME} in WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi + ;; # #----------------------------------------------------------------------- # From 44fc9066ccf7df42ada4014f831f5894e3758fe1 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:13:09 +0000 Subject: [PATCH 03/12] Change CPUs per task to 2 instead of 4 to keep the number of requested nodes from exploding. --- ush/config_defaults.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index e11e2233a..1a9594db5 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1657,7 +1657,7 @@ KMP_AFFINITY_RUN_FCST="scatter" OMP_NUM_THREADS_RUN_FCST="4" OMP_STACKSIZE_RUN_FCST="1024m" -CPUS_PER_TASK_RUN_FCST="4" +CPUS_PER_TASK_RUN_FCST="2" KMP_AFFINITY_RUN_POST="scatter" OMP_NUM_THREADS_RUN_POST="1" From 02012e7f21403a0e4a88c65add8ebac880e7f051 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:17:15 +0000 Subject: [PATCH 04/12] Reduce layout values based on EMC runs on Hera. --- ush/set_predef_grid_params.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 1670f2a83..005b6e4f7 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1190,13 +1190,13 @@ case ${PREDEF_GRID_NAME} in DT_ATMOS="${DT_ATMOS:-36}" - LAYOUT_X="${LAYOUT_X:-40}" - LAYOUT_Y="${LAYOUT_Y:-45}" + LAYOUT_X="${LAYOUT_X:-18}" + LAYOUT_Y="${LAYOUT_Y:-36}" BLOCKSIZE="${BLOCKSIZE:-28}" if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="2" - WRTCMP_write_tasks_per_group="90" + WRTCMP_write_groups=2 + WRTCMP_write_tasks_per_group=72 WRTCMP_output_grid="rotated_latlon" WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}" WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}" From d0479a3932404e8531880f80909a16885cf82816 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:18:11 +0000 Subject: [PATCH 05/12] Remove extra lines. --- ush/setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index f1577516a..a726b5ef2 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2265,8 +2265,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - - # #----------------------------------------------------------------------- # From 49d6e73cdd30ff1b0f704f9ec221a9b713bbce54 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:18:45 +0000 Subject: [PATCH 06/12] Update k/n_split values based on EMC tests (substantially improves run times) --- ush/templates/input.nml.FV3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 64f9f4d05..392689266 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -85,7 +85,7 @@ hord_vt = 6 hydrostatic = .false. io_layout = 1,1 - k_split = 4 + k_split = 5 ke_bg = 0.0 kord_mt = 9 kord_tm = -9 @@ -93,7 +93,7 @@ kord_wz = 9 make_nh = .true. mountain = .false. - n_split = 5 + n_split = 2 n_sponge = 24 n_zs_filter = 0 na_init = 1 From 1b646650bc1ac5c5e093c822cd3dc1bb0eec2bef Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:24:06 +0000 Subject: [PATCH 07/12] Update WRTCMP settings --- ush/set_predef_grid_params.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 005b6e4f7..eabae5d40 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1195,8 +1195,8 @@ case ${PREDEF_GRID_NAME} in BLOCKSIZE="${BLOCKSIZE:-28}" if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups=2 - WRTCMP_write_tasks_per_group=72 + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="144" WRTCMP_output_grid="rotated_latlon" WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}" WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}" From fee99df033d1ad83ff500f99fd6d9d6f06b430c6 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:30:56 +0000 Subject: [PATCH 08/12] Add a WE2E test for the RRFS_NA_3km domain --- .../config.grid_RRFS_NA_3km.sh | 48 +++++++++++++++++++ tests/baselines_list.txt | 2 + 2 files changed, 50 insertions(+) create mode 100644 tests/baseline_configs/config.grid_RRFS_NA_3km.sh diff --git a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh new file mode 100644 index 000000000..a3e98614d --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh @@ -0,0 +1,48 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_NA_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GSD_SAR" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="6" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +######################################################################### +# The following code/namelist/workflow setting changes are necessary to # +# run/optimize end-to-end experiments using the 3-km NA grid # +######################################################################### + +# The model should be built in 32-bit mode (64-bit will result in much longer +# run times. + +# Use k_split=5 and n_split=2, the previous namelist values (k_split=4 and n_split=5) +# will result in significantly longer run times. + +# With 64-bit and k_split=4/n_split=5, time steps with settings below = ~21 seconds +# Changing to k_split=5/n_split=2, time steps with settings below = ~15 seconds +# Changing to 32-bit built, time steps with settings below = ~6 seconds + +NNODES_MAKE_ICS="12" +NNODES_MAKE_LBCS="12" +PPN_MAKE_ICS="4" +PPN_MAKE_LBCS="4" +WTIME_MAKE_LBCS="01:00:00" + +PPN_RUN_FCST="24" + +NNODES_RUN_POST="6" +PPN_RUN_POST="12" + +OMP_STACKSIZE_RUN_FCST="2048m" + +############################################################################### diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 01eae66ad..45050a02d 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -48,6 +48,7 @@ grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_3km grid_RRFS_NA_13km grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 @@ -76,3 +77,4 @@ suite_FV3_GFS_v16beta suite_FV3_GSD_SAR suite_FV3_GSD_v0 suite_FV3_RRFS_v1beta + From 4e9c2ffba085b8cf5ab14887121de0d8df765975 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:40:48 +0000 Subject: [PATCH 09/12] Remove extra line. --- tests/baselines_list.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 45050a02d..b1e725551 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -77,4 +77,3 @@ suite_FV3_GFS_v16beta suite_FV3_GSD_SAR suite_FV3_GSD_v0 suite_FV3_RRFS_v1beta - From db501603fe14dc104f8330180b51fffd5d63efcb Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 03:47:01 +0000 Subject: [PATCH 10/12] Update comments in config.sh file --- tests/baseline_configs/config.grid_RRFS_NA_3km.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh index a3e98614d..bf444964b 100644 --- a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh +++ b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh @@ -22,15 +22,16 @@ USE_USER_STAGED_EXTRN_FILES="TRUE" # run/optimize end-to-end experiments using the 3-km NA grid # ######################################################################### -# The model should be built in 32-bit mode (64-bit will result in much longer -# run times. +# The model should be built in 32-bit mode (64-bit will result in much +# longer run times. -# Use k_split=5 and n_split=2, the previous namelist values (k_split=4 and n_split=5) -# will result in significantly longer run times. +# Use k_split=5 and n_split=2, the previous namelist values (k_split=4 +# and n_split=5) will result in significantly longer run times. -# With 64-bit and k_split=4/n_split=5, time steps with settings below = ~21 seconds -# Changing to k_split=5/n_split=2, time steps with settings below = ~15 seconds -# Changing to 32-bit built, time steps with settings below = ~6 seconds +# For Hera: +# 1) With 64-bit and k_split=4/n_split=5, each time step takes ~21 seconds +# 2) 1) with k_split=5/n_split=2, each time step takes ~15 seconds +# 3) 1) and 2) with 32-bit build, each time step takes ~6 seconds NNODES_MAKE_ICS="12" NNODES_MAKE_LBCS="12" From 6de94f5172128caeb8744a2359e5bed2190b42fd Mon Sep 17 00:00:00 2001 From: jeff beck Date: Tue, 27 Apr 2021 04:11:47 +0000 Subject: [PATCH 11/12] Update to comments. --- tests/baseline_configs/config.grid_RRFS_NA_3km.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh index bf444964b..f9ef409ac 100644 --- a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh +++ b/tests/baseline_configs/config.grid_RRFS_NA_3km.sh @@ -29,9 +29,9 @@ USE_USER_STAGED_EXTRN_FILES="TRUE" # and n_split=5) will result in significantly longer run times. # For Hera: -# 1) With 64-bit and k_split=4/n_split=5, each time step takes ~21 seconds +# 1) With 64-bit build and k_split=4/n_split=5, each time step takes ~21 seconds # 2) 1) with k_split=5/n_split=2, each time step takes ~15 seconds -# 3) 1) and 2) with 32-bit build, each time step takes ~6 seconds +# 3) 2) with 32-bit build, each time step takes ~6 seconds NNODES_MAKE_ICS="12" NNODES_MAKE_LBCS="12" From f996d2ec5341d7f8fcb514d97b51e35dfb450686 Mon Sep 17 00:00:00 2001 From: jeff beck Date: Wed, 5 May 2021 02:37:45 +0000 Subject: [PATCH 12/12] Remove blank lines. --- ush/setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index f1577516a..a726b5ef2 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2265,8 +2265,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - - # #----------------------------------------------------------------------- #