From 3786979aaf89d7ae5efac2ada0cee093a718496a Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Fri, 11 Dec 2020 11:06:53 -0700 Subject: [PATCH 001/203] Feature/update model namelist definitions (#377) ## DESCRIPTION OF CHANGES: Part of the effort described in https://github.com/ufs-community/ufs-srweather-app/pull/71 to update the master branch of ufs-srweather-app to a functioning state. Updates include: - Cherry-picking 5a4b90e7 from release branch in order to include the RRFS_v1alpha CCPP suite, as well as fixing some updated namelist settings in the ufs-weather-model - Fixing other namelist options that have been updated in ufs-weather-model This PR must be merged prior to https://github.com/ufs-community/ufs-srweather-app/pull/71, then that PR updated to point to this PR's hash. ## TESTS CONDUCTED: The following end-to-end tests are now all successful on Hera: - DOT_OR_USCORE - GST_release_public_v1 - get_extrn_files_from_hpss_FV3GFS_FV3GFS - get_extrn_files_from_hpss_GSMGFS_GSMGFS - get_extrn_files_from_hpss_HRRR_RAP - get_extrn_files_from_hpss_RAP_RAP - grid_GSD_HRRR_AK_50km - grid_GSD_RAP13km - grid_RRFS_AK_13km_FV3GFS_FV3GFS - grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta - grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR - grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - new_ESGgrid - new_GFDLgrid - new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE - new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE - regional_001 - regional_002 - suite_FV3_GFS_v15p2 - suite_FV3_GFS_v16beta - suite_FV3_GSD_SAR - suite_FV3_GSD_v0 - suite_FV3_RRFS_v1beta The following end-to-end tests were successful on Jet: - DOT_OR_USCORE - get_extrn_files_from_hpss_FV3GFS_FV3GFS - get_extrn_files_from_hpss_GSMGFS_GSMGFS - get_extrn_files_from_hpss_HRRR_RAP - get_extrn_files_from_hpss_RAP_RAP - grid_RRFS_AK_13km_FV3GFS_FV3GFS - grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta - new_ESGgrid - new_GFDLgrid - new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE - regional_001 - regional_002 - suite_FV3_GFS_v16beta A few tests on Hera and Jet are still failing, but they are known errors and will be fixed at a later date. Cheyenne tests are failing but this is fixed by #378 ## ISSUE (optional): In conjunction with #378 and https://github.com/ufs-community/ufs-srweather-app/pull/71, will solve https://github.com/ufs-community/ufs-srweather-app/issues/57, fixing the master branch of ufs-srweather-app so everyone will be happy and nothing will be broken and we will all live happily ever after. --- scripts/exregional_make_ics.sh | 27 +- scripts/exregional_make_lbcs.sh | 16 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 18 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 18 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 18 + .../config.suite_FV3_RRFS_v1alpha.sh | 19 + tests/baselines_list.txt | 4 + tests/testlist.release_public_v1.txt | 8 +- ush/templates/FV3.input.yml | 44 ++- ush/templates/diag_table.FV3_RRFS_v1alpha | 340 ++++++++++++++++++ ush/templates/field_table.FV3_RRFS_v1alpha | 65 ++++ ush/templates/input.nml.FV3 | 3 +- ush/valid_param_vals.sh | 1 + 13 files changed, 540 insertions(+), 41 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh create mode 100644 tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh create mode 100644 ush/templates/diag_table.FV3_RRFS_v1alpha create mode 100644 ush/templates/field_table.FV3_RRFS_v1alpha diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index d511475ece..0e0ba465a0 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -159,6 +159,7 @@ case "${CCPP_PHYS_SUITE}" in # "FV3_GSD_v0" | \ "FV3_GSD_SAR" | \ + "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" ) if [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ]; then @@ -325,8 +326,9 @@ case "${EXTRN_MDL_NAME_ICS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" else print_err_msg_exit "\ @@ -357,8 +359,9 @@ case "${EXTRN_MDL_NAME_ICS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" else print_err_msg_exit "\ @@ -410,11 +413,11 @@ case "${EXTRN_MDL_NAME_ICS}" in # if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then nsoill_out="4" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then @@ -450,9 +453,10 @@ case "${EXTRN_MDL_NAME_ICS}" in if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then nsoill_out="4" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then @@ -491,8 +495,9 @@ case "${EXTRN_MDL_NAME_ICS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" fi diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 17cc41a1d2..7df3ce5bef 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -159,6 +159,7 @@ case "${CCPP_PHYS_SUITE}" in # "FV3_GSD_v0" | \ "FV3_GSD_SAR" | \ + "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" ) if [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ]; then @@ -294,8 +295,9 @@ case "${EXTRN_MDL_NAME_LBCS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" else print_err_msg_exit "\ @@ -319,8 +321,9 @@ case "${EXTRN_MDL_NAME_LBCS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" else print_err_msg_exit "\ @@ -378,8 +381,9 @@ case "${EXTRN_MDL_NAME_LBCS}" in [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then thomp_mp_climo_file="" elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" else print_err_msg_exit "\ diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh new file mode 100644 index 0000000000..4f8a01107b --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +GRID_GEN_METHOD="ESGgrid" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh new file mode 100644 index 0000000000..e517b6e3b6 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +GRID_GEN_METHOD="ESGgrid" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh new file mode 100644 index 0000000000..90266a61ec --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +GRID_GEN_METHOD="ESGgrid" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh b/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh new file mode 100644 index 0000000000..9383604dcd --- /dev/null +++ b/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +GRID_GEN_METHOD="ESGgrid" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="6" + +DATE_FIRST_CYCL="20190520" +DATE_LAST_CYCL="20190520" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index e2eb50f796..871929a5ac 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -17,14 +17,17 @@ grid_RRFS_AK_3km_RAP_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS grid_RRFS_SUBCONUS_3km_HRRR_RAP @@ -46,4 +49,5 @@ suite_FV3_GFS_v15p2 suite_FV3_GFS_v16beta suite_FV3_GSD_SAR suite_FV3_GSD_v0 +suite_FV3_RRFS_v1alpha suite_FV3_RRFS_v1beta diff --git a/tests/testlist.release_public_v1.txt b/tests/testlist.release_public_v1.txt index 9643771f05..662e3408b4 100644 --- a/tests/testlist.release_public_v1.txt +++ b/tests/testlist.release_public_v1.txt @@ -1,7 +1,7 @@ GST_release_public_v1 -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha +grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index c9121dd642..db2d10d90a 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -11,11 +11,14 @@ FV3_GSD_v0: + gfs_physics_nml: + effr_in: True FV3_GSD_SAR: gfs_physics_nml: &gsd_sar_phys cnvcld: false cnvgwd: false + effr_in: True imfdeepcnv: 0 imfshalcnv: 0 iopt_alb: 2 @@ -33,28 +36,29 @@ FV3_GSD_SAR: shal_cnv: false do_deep: False -FV3_GSD_SAR_v1: - gfs_physics_nml: - <<: *gsd_sar_phys - lsm: 1 - lsoil_lsm: 4 - -FV3_RRFS_v0: +FV3_RRFS_v1alpha: gfs_physics_nml: <<: *gsd_sar_phys + do_deep: False + do_mynnsfclay: False + effr_in: True + imfdeepcnv: -1 + imfshalcnv: -1 lsm: 2 lsoil_lsm: 4 - do_deep: False + atmos_model_nml: + fdiag: 3 FV3_RRFS_v1beta: gfs_physics_nml: <<: *gsd_sar_phys + do_deep: False + do_mynnsfclay: True + effr_in: True + imfdeepcnv: -1 + imfshalcnv: -1 lsm: 2 lsoil_lsm: 4 - imfshalcnv: -1 - IMFDEEPCNV: -1 - DO_MYNNSFCLAY: true - do_deep: False atmos_model_nml: fdiag: 3 @@ -87,7 +91,8 @@ FV3_GFS_2017_gfdlmp: do_deep: False do_mynnedmf: !!python/none do_mynnsfclay: !!python/none - do_sfcperts: !!python/none + lndp_type: !!python/none + n_var_lndp: !!python/none effr_in: True fhcyc: 0.0 fhlwr: 3600.0 @@ -167,7 +172,8 @@ FV3_GFS_2017_gfdlmp_regional: gfs_physics_nml: <<: *gfs_2017_gfdlmp_phys cplflx: False - do_sfcperts: False + lndp_type: 0 + n_var_lndp: 0 iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -321,7 +327,8 @@ FV3_GFS_v16beta: cdmbgwd: [4.0, 0.15, 1.0, 1.0] do_myjpbl: !!python/none do_myjsfc: !!python/none - do_sfcperts: !!python/none + lndp_type: !!python/none + n_var_lndp: !!python/none do_tofd: True do_ysu: !!python/none effr_in: True @@ -329,8 +336,7 @@ FV3_GFS_v16beta: iaer: 5111 icliq_sw: 2 iopt_dveg: 1 - iovr_lw: 3 - iovr_sw: 3 + iovr: 3 isatmedmf: 1 lgfdlmprad: True lheatstrg: True @@ -389,7 +395,8 @@ FV3_CPT_v0: do_sb_physics: True do_mynnedmf: !!python/none do_mynnsfclay: !!python/none - do_sfcperts: !!python/none + lndp_type: !!python/none + n_var_lndp: !!python/none do_shoc: False effr_in: True fhcyc: 0.0 @@ -441,6 +448,5 @@ FV3_CPT_v0: xkzm_h: 0.5 xkzm_m: 0.5 xkzm_s: 1.0 - nam_sfcperts: !!python/none surf_map_nml: !!python/none diff --git a/ush/templates/diag_table.FV3_RRFS_v1alpha b/ush/templates/diag_table.FV3_RRFS_v1alpha new file mode 100644 index 0000000000..99034ff0a9 --- /dev/null +++ b/ush/templates/diag_table.FV3_RRFS_v1alpha @@ -0,0 +1,340 @@ +{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional +{{ starttime.strftime("%Y %m %d %H %M %S") }} + +"grid_spec", -1, "months", 1, "days", "time" +"atmos_static", -1, "hours", 1, "hours", "time" +#"atmos_4xdaily", 1, "hours", 1, "days", "time" +"fv3_history", 1, "years", 1, "hours", "time" +"fv3_history2d", 1, "years", 1, "hours", "time" + +# +#======================= +# ATMOSPHERE DIAGNOSTICS +#======================= +### +# grid_spec +### + "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, + "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, +### +# 4x daily output +### +# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 +### +# gfs static data +### + "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 + "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 + "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 + "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 + "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 +### +# FV3 variabls needed for NGGPS evaluation +### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 + +"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 + +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +# Aerosols (CCN, IN) from Thompson microphysics +"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 +# Cloud effective radii from Thompson and WSM6 microphysics +"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 +# Prognostic/diagnostic variables from MYNN +"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 +# Prognostic/diagnostic variables from RUC LSM +"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/field_table.FV3_RRFS_v1alpha b/ush/templates/field_table.FV3_RRFS_v1alpha new file mode 100644 index 0000000000..0a927de455 --- /dev/null +++ b/ush/templates/field_table.FV3_RRFS_v1alpha @@ -0,0 +1,65 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic graupel mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water number concentration + "TRACER", "atmos_mod", "water_nc" + "longname", "cloud liquid water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# water- and ice-friendly aerosols (Thompson) + "TRACER", "atmos_mod", "liq_aero" + "longname", "water-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / + "TRACER", "atmos_mod", "ice_aero" + "longname", "ice-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=1.e30" / diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 75cc97c8f3..f6890ae008 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -139,7 +139,8 @@ debug = .false. do_mynnedmf = .true. do_mynnsfclay = .false. - do_sfcperts = .false. + lndp_type = 0 + n_var_lndp = 0 do_shum = .false. do_skeb = .false. do_sppt = .false. diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 008e8058e0..f94a1569d2 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -30,6 +30,7 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GFS_v15p2" \ "FV3_GFS_v16beta" \ "FV3_RRFS_v1beta" \ +"FV3_RRFS_v1alpha" \ ) valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") From d889b0e1710744676b34c63e77e5b5288574e168 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Fri, 11 Dec 2020 11:17:01 -0700 Subject: [PATCH 002/203] Cherry-pick #343 from release branch: Allow running from crontab on Cheyenne, as well as fix workflow generation on Cheyenne (#378) ## DESCRIPTION OF CHANGES: A fix was applied to the release branch for the Cheyenne workflow and automated testing in #343, this PR applies the same to develop. ## TESTS CONDUCTED: Workflow built and ran on Cheyenne; the following tests were run and all passed: - grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 ## ISSUE (optional): Fully resolves #326 --- ush/fill_jinja_template.py | 2 +- ush/launch_FV3LAM_wflow.sh | 15 ++++++++------- ush/templates/FV3LAM_wflow.xml | 6 +++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ush/fill_jinja_template.py b/ush/fill_jinja_template.py index ef3b57245c..55cf0a704c 100755 --- a/ush/fill_jinja_template.py +++ b/ush/fill_jinja_template.py @@ -259,7 +259,7 @@ def main(cla): tvars = {} for var in template_vars: - if cfg.get(var) is None: + if cfg.get(var, "NULL") == "NULL": raise KeyError(f'{var} does not exist in user-supplied settings!') if not cla.quiet: diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 60910d812d..14cb1ad19b 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -91,13 +91,14 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -if [ "$MACHINE" != "CHEYENNE" ]; then - if [ "$MACHINE" = "ORION" ]; then - module load contrib rocoto - else - module purge - module load rocoto - fi +if [ "$MACHINE" = "CHEYENNE" ]; then + module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ + module load rocoto +elif [ "$MACHINE" = "ORION" ]; then + module load contrib rocoto +else + module purge + module load rocoto fi # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index f95c0ecd27..2dab388cbf 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -64,17 +64,17 @@ the "HPSS" type is used for the GET_EXTRN_ICS_TN and GET_EXTRN_LBCS_TN tasks; and the "FCST" type is used for the RUN_FCST_TN task. --> -{%- if (partition_default != "") %} +{%- if partition_default is not none %} &ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}"> {%- else %} &ACCOUNT;&QUEUE_DEFAULT;"> {%- endif %} -{%- if (partition_hpss != "") %} +{%- if partition_hpss is not none %} &ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}"> {%- else %} &ACCOUNT;&QUEUE_HPSS;"> {%- endif %} -{%- if (partition_fcst != "") %} +{%- if partition_fcst is not none %} &ACCOUNT;&QUEUE_FCST;{{ partition_fcst }}"> {%- else %} &ACCOUNT;&QUEUE_FCST;"> From e04bb1366fde52de14b10c3cdd59c8465f3e4bad Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 14 Dec 2020 11:00:40 -0700 Subject: [PATCH 003/203] Maximize RRFS_CONUS_[25,13,3]km grids so they span as much of HRRR CONUS domain as possible (#360) ## DESCRIPTION OF CHANGES: ### Main changes: * Change grid parameters for the RRFS_COUNS_25km, RRFS_CONUS_13km, and RRFS_CONUS_3km grids so that they span as much of the HRRR CONUS domain as possible. Note that this is done without regard to whether NX and NY are evenly divisible by LAYOUT_X and LAYOUT_Y because these restrictions were recently removed from the forecast model. * Remove several checks on the grid parameters. These are: (1) The check on whether NX and NY are evenly divisible by LAYOUT_X and LAYOUT_Y, respectively. This restriction has been removed from the forecast model. (2) The check on BLOCKSIZE, i.e. whether the quantity (NX/LAYOUT_X)*(NY/LAYOUT_Y) is evenly divisible by BLOCKSIZE. This already depends on restriction (1) being satisfied, so it cannot be satisfied if (1) is not satisfied. (3) The check on whether NY is evenly divisible by the number of write-component tasks per write-component group. Tests were run with parameters that violated this test, and the model did not crash, so we assume this is no longer a restriction. * Modify NCL plotting scripts to make them work with the latest version of the workflow. ### Bug fixes: * Fix bug in set_predef_grid_params.sh involving an extra space after the equal sign in the lines that set GRID_GEN_METHOD. * Remove the FV3_RRFS_v1alpha suite from the list of WE2E tests because that suite is not (yet?) supported by the version of ufs-weather-model this PR is using. * Fix bug in run_experiment.sh where the variable FV3GFS_FILE_FMT_ICS should be replaced with FV3GFS_FILE_FMT_LBCS. ### Improvements: * Forecast model executable: (1) Remove section of code that copies the FV3 executable from its location in the ufs-weather-model directory to the bin directory for all other executables. This is no longer necessary since the new build system already places this executable in the bin directory. (2) Change the name of the forecast model executable to the latest one that is being used in ufs-weather-model. * Remove commented-out code that was previously used to create module files for the various workflow tasks. * Change the default physics suite to GFS_v15p2 to be consistent with the release/public-v1 branch. * Alphabetize namelist variable entries in FV3.input.yml and input.nml.FV3. * Edit comments. ## TESTS CONDUCTED: Ran all 49 WE2E tests on Hera. All except 5 tests succeeded. The results are as follows: ``` (1) DOT_OR_USCORE SUCCESS (2) GST_release_public_v1 SUCCESS (3) community_ensemble_008mems SUCCESS (4) community_ensemble_2mems SUCCESS (5) get_extrn_files_from_hpss_FV3GFS_FV3GFS SUCCESS (6) get_extrn_files_from_hpss_GSMGFS_GSMGFS SUCCESS (7) get_extrn_files_from_hpss_HRRR_RAP SUCCESS (8) get_extrn_files_from_hpss_RAP_RAP SUCCESS (9) grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS SUCCESS (10) grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS SUCCESS (11) grid_GSD_HRRR_AK_50km SUCCESS (12) grid_GSD_RAP13km FAILURE (13) grid_RRFS_AK_13km_FV3GFS_FV3GFS SUCCESS (14) grid_RRFS_AK_13km_RAP_RAP FAILURE (15) grid_RRFS_AK_3km_FV3GFS_FV3GFS SUCCESS (16) grid_RRFS_AK_3km_RAP_RAP FAILURE (17) grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 SUCCESS (18) grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta SUCCESS (19) grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR SUCCESS (20) grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta SUCCESS (21) grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 SUCCESS (22) grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta SUCCESS (23) grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR SUCCESS (24) grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta SUCCESS (25) grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 SUCCESS (26) grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta SUCCESS (27) grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR SUCCESS (28) grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta SUCCESS (29) grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS SUCCESS (30) grid_RRFS_SUBCONUS_3km_HRRR_RAP FAILURE (31) nco_CONUS_25km_GFDLgrid SUCCESS (32) nco_RRFS_CONUS_25km_HRRR_RAP SUCCESS (33) nco_RRFS_CONUS_3km_FV3GFS_FV3GFS SUCCESS (34) nco_RRFS_CONUS_3km_HRRR_RAP SUCCESS (35) nco_RRFS_SUBCONUS_3km_HRRR_RAP SUCCESS (36) nco_ensemble SUCCESS (37) new_ESGgrid SUCCESS (38) new_GFDLgrid SUCCESS (39) new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE SUCCESS (40) new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE SUCCESS (41) pregen_grid_orog_sfc_climo SUCCESS (42) regional_001 SUCCESS (43) regional_002 SUCCESS (44) suite_FV3_CPT_v0 FAILURE (45) suite_FV3_GFS_v15p2 SUCCESS (46) suite_FV3_GFS_v16beta SUCCESS (47) suite_FV3_GSD_SAR SUCCESS (48) suite_FV3_GSD_v0 SUCCESS (49) suite_FV3_RRFS_v1beta SUCCESS ``` The 5 failed tests (with information on the failures) are as follows: ``` (12) grid_GSD_RAP13km Failed in most of the run_post tasks. (14) grid_RRFS_AK_13km_RAP_RAP Failed in the run_fcst task with "AVOST IN VILKA" messages. (16) grid_RRFS_AK_3km_RAP_RAP Failed in the run_fcst task with "AVOST IN VILKA" messages. (30) grid_RRFS_SUBCONUS_3km_HRRR_RAP Failed in the run_fcst task without a clear failure message. (44) suite_FV3_CPT_v0 Failed in the run_fcst task with the following namelist read error: forrtl: severe (19): invalid reference to variable in NAMELIST input, unit -5, file Internal Formatted NML Read, line -1, position 13 ``` Except for FV3_CPT_v0, the failed tests do not use the new RRFS_CONUS grids. Thus, these failures are not related to the CONUS grid modifications. The failure of FV3_CPT_v0 is likely due to a bad namelist variable specification and thus also unrelated to the CONUS grid modifications. Note that all tests using grids for which NX and NY are not evenly divisible by LAYOUT_X and LAYOUT_Y, respectively (which include all 3 new versions of the RRFS_CONUS grids), and thus the quantity (NX/LAYOUT_X)*(NY/LAYOUT_Y) is not evenly divisible by BLOCKSIZE gave warnings of this kind in the log file for the run_fcst task: `WARNING from PE 478: atmos_modeldefine_blocks_packed: domain ( 59 65) is not an even divisor with definition ( 32) - blocks will not be uniform with a remainder of 27 ` This warning should either be removed (since this restriction on BLOCKSIZE has been removed from the code) or, if having this remainder makes the model run less efficiently, be replaced by a different caution message. --- tests/baselines_list.txt | 4 - tests/run_experiments.sh | 4 +- ...calc_wrtcmp_grid_params_lambert_cnfrml.ncl | 22 ++-- ush/NCL/find_wrtcmp_grid_params.ncl | 29 ++--- ush/NCL/plot_grid.ncl | 39 +++--- ush/config_defaults.sh | 12 +- ush/generate_FV3LAM_wflow.sh | 118 +----------------- ush/set_predef_grid_params.sh | 112 ++++++++--------- ush/setup.sh | 86 ------------- ush/templates/FV3.input.yml | 18 +-- ush/templates/input.nml.FV3 | 4 +- 11 files changed, 129 insertions(+), 319 deletions(-) diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 871929a5ac..e2eb50f796 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -17,17 +17,14 @@ grid_RRFS_AK_3km_RAP_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS grid_RRFS_SUBCONUS_3km_HRRR_RAP @@ -49,5 +46,4 @@ suite_FV3_GFS_v15p2 suite_FV3_GFS_v16beta suite_FV3_GSD_SAR suite_FV3_GSD_v0 -suite_FV3_RRFS_v1alpha suite_FV3_RRFS_v1beta diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 2e841b1120..90539bb42d 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -748,10 +748,10 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) fi elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ diff --git a/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl b/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl index 2a917a1261..c4c589b0df 100644 --- a/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl +++ b/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl @@ -16,7 +16,7 @@ loadscript(lib_location + "convert_lambert_cnfrml_coords_to_sphr.ncl") undef("calc_wrtcmp_grid_params_lambert_cnfrml") function calc_wrtcmp_grid_params_lambert_cnfrml( \ - wrtcmp_config_fn:string, \ + model_config_tmpl_fp:string, \ lon_ctr_native:snumeric, lat_ctr_native:snumeric, \ lon_tile_corners_face_midpts_native[8]:snumeric, \ lat_tile_corners_face_midpts_native[8]:snumeric, \ @@ -244,16 +244,16 @@ begin param_value := char_sq + "lambert_conformal" + char_sq param_value_str := tostring(param_value) - regex_search = "^(\s*" + param_name + ":\s+)(<" + param_name + ">)(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)('\{\{ " + param_name + " \}\}')(\s*)(.*)" regex_print = "\1" + param_value_str sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp param_names_and_vals(np) = systemfunc(sed_cmd) regex_print = "\4" sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp trailing_comments(np) = systemfunc(sed_cmd) ; ; ********************************************************************** @@ -326,22 +326,24 @@ begin ; ; Set the regular expression to search for. ; - regex_search = "^(\s*" + param_name + ":\s+)(<" + param_name + ">)(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)(\{\{ " + param_name + " \}\})(\s*)(.*)" ; ; Get the parameter name and value without the trailing comment (if any). ; regex_print = "\1" + param_value_str sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn - param_names_and_vals(np) = systemfunc(sed_cmd) + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp + tmp := systemfunc(sed_cmd) + param_names_and_vals(np) = tmp(0) ; ; Get the trailing name and comment. ; regex_print = "\4" sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn - trailing_comments(np) = systemfunc(sed_cmd) - + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp + tmp := systemfunc(sed_cmd) + trailing_comments(np) = tmp(0) + end do ; ; ********************************************************************** diff --git a/ush/NCL/find_wrtcmp_grid_params.ncl b/ush/NCL/find_wrtcmp_grid_params.ncl index d7b3d97c98..fce1627f8e 100644 --- a/ush/NCL/find_wrtcmp_grid_params.ncl +++ b/ush/NCL/find_wrtcmp_grid_params.ncl @@ -22,8 +22,13 @@ expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/ex expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_13km_HRRRX_RAPX" expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_3km_FV3GFS_FV3GFS" expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try03/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" -expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try03/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try03/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" + +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" + +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" ; ; ********************************************************************** @@ -100,7 +105,7 @@ begin var_defns_fp = expt_dir + "/var_defns.sh" param_names = (/ \ -"WRTCMP_PARAMS_TMPL_FP", \ +"MODEL_CONFIG_TMPL_FP", \ "WRTCMP_output_grid", \ "GTYPE", \ "CRES", \ @@ -125,8 +130,8 @@ print("sed_output = " + sed_output) ; Convert the output from the sed command (which will be a string) to ; the appropriate NCL data type. ; - if (strcmp_exact(param_name, "WRTCMP_PARAMS_TMPL_FP")) then - WRTCMP_PARAMS_TMPL_FP = tostring(sed_output) + if (strcmp_exact(param_name, "MODEL_CONFIG_TMPL_FP")) then + MODEL_CONFIG_TMPL_FP = tostring(sed_output) else if (strcmp_exact(param_name, "WRTCMP_output_grid")) then WRTCMP_output_grid = tostring(sed_output) else if (strcmp_exact(param_name, "GTYPE")) then @@ -226,18 +231,6 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") ; ; ********************************************************************** ; -; Set the full path to the file that will contain a copy of the write- -; component template file with placeholders replaced with actual values. -; Then copy the template file to that file. -; -; ********************************************************************** -; - wrtcmp_config_fp = expt_dir + "/aaaa" - sys_output = systemfunc("cp " + WRTCMP_PARAMS_TMPL_FP + \ - " " + wrtcmp_config_fp) -; -; ********************************************************************** -; ; Extract and save into new, appropriately dimensioned variables the ; output from the grid-read operation above. Then call the function ; that calculates the write-component grid parameters for a lambert- @@ -267,7 +260,7 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") if (strcmp_exact(WRTCMP_output_grid, "rotated_latlon")) then out := calc_wrtcmp_grid_params_rotated_latlon( \ - wrtcmp_config_fp, \ + MODEL_CONFIG_TMPL_FP, \ lon_grid_cntr_native, lat_grid_cntr_native, \ lon_tile_corners_face_midpts_native, \ lat_tile_corners_face_midpts_native, \ @@ -277,7 +270,7 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") else if (strcmp_exact(WRTCMP_output_grid, "lambert_conformal")) then out := calc_wrtcmp_grid_params_lambert_cnfrml( \ - wrtcmp_config_fp, \ + MODEL_CONFIG_TMPL_FP, \ lon_grid_cntr_native, lat_grid_cntr_native, \ lon_tile_corners_face_midpts_native, \ lat_tile_corners_face_midpts_native, \ diff --git a/ush/NCL/plot_grid.ncl b/ush/NCL/plot_grid.ncl index 5c5f903b8b..d33466d46a 100644 --- a/ush/NCL/plot_grid.ncl +++ b/ush/NCL/plot_grid.ncl @@ -77,20 +77,26 @@ CDATE="2019070100" ;CDATE="2019070100" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/grid_CONUS_3km_GFDLgrid" -;CDATE="2019070100" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try02/grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +CDATE="2019070100" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/grid_CONUS_25km_GFDLgrid" -expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try03/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" -CDATE="2019070100" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_bugfix_adjust_CONUS_grids/expt_dirs/try03/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +;expt_dir = "/work/noaa/gsd-fv3-dev/gsketefia/UFS/PR_release_branch/expt_dirs/try02/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try02/grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +;CDATE="2019070100" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/suite_FV3_GSD_SAR" ;CDATE="2019052000" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/grid_RRFS_AK_3km" +;expt_dir = "/work/noaa/gsd-fv3-dev/gsketefia/UFS/PR_release_branch/expt_dirs/try01/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try02/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" ;CDATE="2019052000" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/grid_RRFS_AK_13km" ;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_predef_grids/expt_dirs/try01/grid_RRFS_AK_13km_old001" +;expt_dir = "/work/noaa/gsd-fv3-dev/gsketefia/UFS/PR_release_branch/expt_dirs/try02/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" ;CDATE="2019052000" @@ -136,6 +142,7 @@ RAP_bdy_color = "red" RAP_dir = "/scratch3/BMC/fim/Gerard.Ketefian/regional_FV3_EMC_visit_20180509" RAP_dir = "/scratch1/BMC/gsd-fv3-dev/Gerard.Ketefian/regional_FV3_EMC_visit_20180509" +;RAP_dir = "/work/noaa/gsd-fv3-dev/gsketefia/UFS" ; This is on Orion. ;RAP_grid_fn = "/home/Gerard/fv3_regional/fv3sar_workflow/ush/NCL/plot_grid.ncl" ;RAP_grid_fn = "/scratch3/BMC/fim/Gerard.Ketefian/regional_FV3_EMC_visit_20180509/geo_em.d01.RAP.nc" RAP_grid_fn = RAP_dir + "/geo_em.d01.RAP.nc" @@ -217,21 +224,21 @@ subreg_limits := (/ -135, -60, 20, 55 /) ; HRRR domain if using "cyln" map proje ;subreg_limits := (/ -122.85, -122.83, 21.49, 21.51 /) ; 0.02deg x 0.02deg ; Zoom in on southeast corner of CONUS domain. -;subreg_limits := (/ -68, -88, 22, 42 /) ; 20deg x 20deg -;subreg_limits := (/ -70, -80, 20, 30 /) ; 10deg x 10deg -;subreg_limits := (/ -71, -76, 21, 26 /) ; 5deg x 5deg -;subreg_limits := (/ -71.9, -73.9, 21.4, 23.4 /) ; 2deg x 2deg -;subreg_limits := (/ -72, -73, 21.2, 22.2 /) ; 1deg x 1deg -;subreg_limits := (/ -72.0, -72.5, 21.45, 21.95 /) ; 0.5deg x 0.5deg -;subreg_limits := (/ -72.1, -72.2, 21.45, 21.55 /) ; 0.1deg x 0.1deg -;subreg_limits := (/ -72.15, -72.17, 21.49, 21.51 /) ; 0.02deg x 0.02deg +;subreg_limits := (/ -88, -68, 18, 38 /) ; 20deg x 20deg +;subreg_limits := (/ -80, -70, 20, 30 /) ; 10deg x 10deg +;subreg_limits := (/ -76, -71, 21, 26 /) ; 5deg x 5deg +;subreg_limits := (/ -73.9, -71.9, 21.0, 23.0 /) ; 2deg x 2deg +;subreg_limits := (/ -73, -72, 21.2, 22.2 /) ; 1deg x 1deg +;subreg_limits := (/ -72.5, -72.0, 21.45, 21.95 /) ; 0.5deg x 0.5deg +;subreg_limits := (/ -72.25, -72.15, 21.45, 21.55 /) ; 0.1deg x 0.1deg +;subreg_limits := (/ -72.17, -72.15, 21.49, 21.51 /) ; 0.02deg x 0.02deg ; Zoom in on northeast corner of CONUS domain. -;subreg_limits := (/ -66, -86, 32, 52 /) ; 20deg x 20deg -;subreg_limits := (/ -64, -74, 40, 50 /) ; 10deg x 10deg. -;subreg_limits := (/ -62, -67, 44, 49 /) ; 5deg x 5deg. -;subreg_limits := (/ -61.5, -63.5, 46.2, 48.2 /) ; 2deg x 2deg. -;subreg_limits := (/ -61.2, -62.2, 47, 48 /) ; 1deg x 1deg. +;subreg_limits := (/ -78, -58, 32, 52 /) ; 20deg x 20deg +;subreg_limits := (/ -70, -60, 40, 50 /) ; 10deg x 10deg. +;subreg_limits := (/ -65.5, -60.5, 44, 49 /) ; 5deg x 5deg. +;subreg_limits := (/ -63.0, -61.0, 46.2, 48.2 /) ; 2deg x 2deg. +;subreg_limits := (/ -62.2, -61.2, 47, 48 /) ; 1deg x 1deg. diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 2babf3c3c5..c9f284eb7f 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -330,7 +330,7 @@ FV3_NML_YAML_CONFIG_FN="FV3.input.yml" FV3_NML_BASE_ENS_FN="input.nml.base_ens" MODEL_CONFIG_FN="model_configure" NEMS_CONFIG_FN="nems.configure" -FV3_EXEC_FN="fv3_gfs.x" +FV3_EXEC_FN="ufs_model" WFLOW_XML_FN="FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN="var_defns.sh" @@ -476,7 +476,7 @@ EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) # #----------------------------------------------------------------------- # -CCPP_PHYS_SUITE="FV3_GSD_v0" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" # #----------------------------------------------------------------------- # @@ -720,6 +720,14 @@ ESGgrid_WIDE_HALO_WIDTH="6" # the frequency with which the top level routine in the dynamics is called # as well as the frequency with which the physics is called." # +# Note that if using one of the predefined grids (i.e. if PREDEF_GRID_NAME +# is not set to a null string, either below or in the custom workflow +# configuration file specified by EXPT_CONFIG_FN), then DT_ATMOS is +# overwritten by the value for that predefined grid. The predefined +# grid parameters are specified in the script +# +# $HOMErrfs/ush/set_predef_grid_params.sh +# #----------------------------------------------------------------------- # DT_ATMOS="18" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index f88d380869..48453ac232 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -330,58 +330,6 @@ done # #----------------------------------------------------------------------- # -# For select workflow tasks, copy the system-specific environment and -# module settings README file from the ufs-srweather-app repository to -# the appropriate subdirectory under the workflow directory tree. In -# principle, sourcing this file is better than having hard-coded module -# files for tasks because the copied module files will always be -# up-to-date. -# -#----------------------------------------------------------------------- -# -#machine=${MACHINE,,} - -#cd_vrfy "${MODULES_DIR}/tasks/$machine" - -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${MAKE_GRID_TN}" -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${MAKE_OROG_TN}" -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${MAKE_SFC_CLIMO_TN}" -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${MAKE_ICS_TN}" -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${MAKE_LBCS_TN}" -#if [ $MACHINE = "WCOSS_CRAY" -o $MACHINE = "WCOSS_DELL_P3" ] ; then -# cp_vrfy -f "${UFS_WTHR_MDL_DIR}/modulefiles/${machine}/fv3" "${RUN_FCST_TN}" -#else -# cp_vrfy -f "${UFS_WTHR_MDL_DIR}/modulefiles/${machine}.intel/fv3" "${RUN_FCST_TN}" -#fi -#cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${RUN_FCST_TN}" - -task_names=( "${MAKE_GRID_TN}" "${MAKE_OROG_TN}" "${MAKE_SFC_CLIMO_TN}" "${MAKE_ICS_TN}" "${MAKE_LBCS_TN}" "${RUN_FCST_TN}" ) -# -# Only some platforms build EMC_post using modules, and some machines -# require a different EMC_post modulefile name. -# -#if [ "${MACHINE}" = "CHEYENNE" ]; then -# print_info_msg "No post modulefile needed for ${MACHINE}" -#elif [ "${MACHINE}" = "WCOSS_CRAY" ]; then -# cp_vrfy -f "${EMC_POST_DIR}/modulefiles/post/v8.0.0-cray-intel" "${RUN_POST_TN}" -# cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${RUN_POST_TN}" -# task_names+=("${RUN_POST_TN}") -#else -# cp_vrfy -f "${SR_WX_APP_TOP_DIR}/docs/README_${machine}_intel.txt" "${RUN_POST_TN}" -# task_names+=("${RUN_POST_TN}") -#fi - -#for task in "${task_names[@]}"; do -# modulefile_local="${task}.local" -# if [ -f ${modulefile_local} ]; then -# cat "${modulefile_local}" >> "${task}" -# fi -#done - -#cd_vrfy - -# -#----------------------------------------------------------------------- -# # Create a symlink in the experiment directory that points to the workflow # (re)launch script. # @@ -521,68 +469,6 @@ cp_vrfy "${CCPP_PHYS_SUITE_IN_CCPP_FP}" "${CCPP_PHYS_SUITE_FP}" # #----------------------------------------------------------------------- # -# Copy the forecast model executable from its location in the directory -# in which the forecast model repository was cloned (UFS_WTHR_MDL_DIR) -# to the executables directory (EXECDIR). -# -# Note that if there is already an experiment that is running the forecast -# task (so that the forecast model executable in EXECDIR is in use) and -# the user tries to generate another experiment, the generation of this -# second experiment will fail because the operating system won't allow -# the existing executable in EXECDIR to be overwritten (because it is -# "busy", i.e. in use by the first experiment). For this reason, below, -# we try to prevent this situation by comparing the ages of the source -# and target executables and attempting the copy only if the source one -# is newer (or if the target doesn't exist). This will very likely prevent -# the situation described above, but it doesn't guarantee that it will -# never happen (it will still happen if an experiment is running a forecast -# while the user rebuilts the forecast model and attempts to generate a -# new experiment. For this reason, this copy operation should really be -# performed duirng the build step, not here. -# -# Question: -# Why doesn't the build script(s) perform this action? It should... -# -#----------------------------------------------------------------------- -# -exec_fn="ufs_model" -exec_fp="${SR_WX_APP_TOP_DIR}/bin/${exec_fn}" -#Check for the old build location for fv3 executable -if [ ! -f "${exec_fp}" ]; then - exec_fp_alt="${UFS_WTHR_MDL_DIR}/build/${exec_fn}" - if [ ! -f "${exec_fp_alt}" ]; then - print_err_msg_exit "\ -The executable (exec_fp) for running the forecast model does not exist: - exec_fp = \"${exec_fp}\" -Please ensure that you've built this executable." - else - exec_fp="${exec_fp_alt}" - fi -fi - -if [ ! -f "${exec_fp}" ]; then - print_err_msg_exit "\ -The executable (exec_fp) for running the forecast model does not exist: - exec_fp = \"${exec_fp}\" -Please ensure that you've built this executable." -fi -# -# Make a copy of the executable in the executables directory only if a -# copy doens't already exist or if a copy does exist but is older than -# the original. -# -if [ ! -e "${FV3_EXEC_FP}" ] || \ - [ "${exec_fp}" -nt "${FV3_EXEC_FP}" ]; then - print_info_msg "$VERBOSE" " -Copying the FV3-LAM executable (exec_fp) to the executables directory -(EXECDIR): - exec_fp = \"${exec_fp}\" - EXECDIR = \"$EXECDIR\"" - cp_vrfy "${exec_fp}" "${FV3_EXEC_FP}" -fi -# -#----------------------------------------------------------------------- -# # Set parameters in the FV3-LAM namelist file. # #----------------------------------------------------------------------- @@ -916,6 +802,10 @@ fi { restore_shell_opts; } > /dev/null 2>&1 } + + + + # #----------------------------------------------------------------------- # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 67d68fc689..b2ce52ed2f 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -10,7 +10,7 @@ function set_predef_grid_params() { # #----------------------------------------------------------------------- # -# Get the full path to the file in which this script/function is located +# Get the full path to the file in which this script/function is located # (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in # which the file is located (scrfunc_dir). # @@ -31,7 +31,7 @@ local func_name="${FUNCNAME[0]}" #----------------------------------------------------------------------- # # Set grid and other parameters according to the value of the predefined -# domain (PREDEF_GRID_NAME). Note that the code will enter this script +# domain (PREDEF_GRID_NAME). Note that the code will enter this script # only if PREDEF_GRID_NAME has a valid (and non-empty) value. # #################### @@ -84,7 +84,7 @@ case ${PREDEF_GRID_NAME} in # "RRFS_CONUS_25km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-97.5 ESGgrid_LAT_CTR=38.5 @@ -92,8 +92,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="25000.0" ESGgrid_DELY="25000.0" - ESGgrid_NX=200 - ESGgrid_NY=112 + ESGgrid_NX=202 + ESGgrid_NY=116 ESGgrid_WIDE_HALO_WIDTH=6 @@ -111,10 +111,10 @@ case ${PREDEF_GRID_NAME} in WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="197" - WRTCMP_ny="107" - WRTCMP_lon_lwr_left="-121.12455072" - WRTCMP_lat_lwr_left="23.89394570" + WRTCMP_nx="199" + WRTCMP_ny="111" + WRTCMP_lon_lwr_left="-121.23349066" + WRTCMP_lat_lwr_left="23.41731593" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" fi @@ -128,7 +128,7 @@ case ${PREDEF_GRID_NAME} in # "RRFS_CONUS_13km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-97.5 ESGgrid_LAT_CTR=38.5 @@ -136,15 +136,15 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX=384 - ESGgrid_NY=224 + ESGgrid_NX=396 + ESGgrid_NY=232 ESGgrid_WIDE_HALO_WIDTH=6 DT_ATMOS="45" LAYOUT_X="16" - LAYOUT_Y="8" + LAYOUT_Y="10" BLOCKSIZE="32" if [ "$QUILTING" = "TRUE" ]; then @@ -155,10 +155,10 @@ case ${PREDEF_GRID_NAME} in WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="381" - WRTCMP_ny="217" - WRTCMP_lon_lwr_left="-121.11782706" - WRTCMP_lat_lwr_left="23.18443988" + WRTCMP_nx="393" + WRTCMP_ny="225" + WRTCMP_lon_lwr_left="-121.70231097" + WRTCMP_lat_lwr_left="22.57417972" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" fi @@ -172,7 +172,7 @@ case ${PREDEF_GRID_NAME} in # "RRFS_CONUS_3km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-97.5 ESGgrid_LAT_CTR=38.5 @@ -180,8 +180,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=1680 - ESGgrid_NY=1024 + ESGgrid_NX=1748 + ESGgrid_NY=1038 ESGgrid_WIDE_HALO_WIDTH=6 @@ -199,10 +199,10 @@ case ${PREDEF_GRID_NAME} in WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="1678" - WRTCMP_ny="1002" - WRTCMP_lon_lwr_left="-121.29805882" - WRTCMP_lat_lwr_left="22.26588883" + WRTCMP_nx="1746" + WRTCMP_ny="1014" + WRTCMP_lon_lwr_left="-122.17364391" + WRTCMP_lat_lwr_left="21.88588562" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" fi @@ -216,7 +216,7 @@ case ${PREDEF_GRID_NAME} in # "RRFS_SUBCONUS_3km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-97.5 ESGgrid_LAT_CTR=35.0 @@ -263,7 +263,7 @@ case ${PREDEF_GRID_NAME} in # "RRFS_AK_13km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-161.5 ESGgrid_LAT_CTR=63.0 @@ -333,23 +333,23 @@ case ${PREDEF_GRID_NAME} in # "RRFS_AK_3km") -# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then +# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # # GFDLgrid_LON_T6_CTR=-160.8 # GFDLgrid_LAT_T6_CTR=63.0 # GFDLgrid_STRETCH_FAC=1.161 # GFDLgrid_RES="768" # GFDLgrid_REFINE_RATIO=4 -# +# # num_margin_cells_T6_left=204 # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) -# +# # num_margin_cells_T6_right=204 # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) -# +# # num_margin_cells_T6_bottom=249 # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) -# +# # num_margin_cells_T6_top=249 # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # @@ -379,7 +379,7 @@ case ${PREDEF_GRID_NAME} in # # elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-161.5 ESGgrid_LAT_CTR=63.0 @@ -428,7 +428,7 @@ case ${PREDEF_GRID_NAME} in # "CONUS_25km_GFDLgrid") - GRID_GEN_METHOD= "GFDLgrid" + GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR=-97.5 GFDLgrid_LAT_T6_CTR=38.5 @@ -483,7 +483,7 @@ case ${PREDEF_GRID_NAME} in # "CONUS_3km_GFDLgrid") - GRID_GEN_METHOD= "GFDLgrid" + GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR=-97.5 GFDLgrid_LAT_T6_CTR=38.5 @@ -575,16 +575,16 @@ case ${PREDEF_GRID_NAME} in # GFDLgrid_STRETCH_FAC=1.0 # ??? # GFDLgrid_RES="768" # GFDLgrid_REFINE_RATIO=3 # ??? -# +# # num_margin_cells_T6_left=61 # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) -# +# # num_margin_cells_T6_right=67 # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) -# +# # num_margin_cells_T6_bottom=165 # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) -# +# # num_margin_cells_T6_top=171 # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # @@ -600,7 +600,7 @@ case ${PREDEF_GRID_NAME} in # # elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar @@ -627,7 +627,7 @@ case ${PREDEF_GRID_NAME} in # 'ksplit' is the factor that determines the timestep for this process (divided # Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the +# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the # top-level routine in the dynamics is called as well as the frequency with which the physics is called. # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh @@ -676,7 +676,7 @@ case ${PREDEF_GRID_NAME} in # "EMC_HI") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/hi/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar @@ -702,7 +702,7 @@ case ${PREDEF_GRID_NAME} in # 'ksplit' is the factor that determines the timestep for this process (divided # Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the +# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the # top-level routine in the dynamics is called as well as the frequency with which the physics is called. # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh @@ -751,7 +751,7 @@ case ${PREDEF_GRID_NAME} in # "EMC_PR") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/pr/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar @@ -777,7 +777,7 @@ case ${PREDEF_GRID_NAME} in # 'ksplit' is the factor that determines the timestep for this process (divided # Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the +# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the # top-level routine in the dynamics is called as well as the frequency with which the physics is called. # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh @@ -826,7 +826,7 @@ case ${PREDEF_GRID_NAME} in # "EMC_GU") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/guam/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar @@ -852,7 +852,7 @@ case ${PREDEF_GRID_NAME} in # 'ksplit' is the factor that determines the timestep for this process (divided # Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the +# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the # top-level routine in the dynamics is called as well as the frequency with which the physics is called. # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh @@ -884,7 +884,7 @@ case ${PREDEF_GRID_NAME} in #Write component grid must always be <= compute grid (without haloes) WRTCMP_nx="420" WRTCMP_ny="348" -#Lower left latlon (southwest corner) Used /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/dbrowse/fv3grid utility to find best value +#Lower left latlon (southwest corner) Used /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/dbrowse/fv3grid utility to find best value WRTCMP_lon_lwr_left="140" WRTCMP_lat_lwr_left="10" WRTCMP_dx="$ESGgrid_DELX" @@ -900,7 +900,7 @@ case ${PREDEF_GRID_NAME} in # "GSL_HAFSV0.A_25km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-62.0 ESGgrid_LAT_CTR=22.0 @@ -942,7 +942,7 @@ case ${PREDEF_GRID_NAME} in # "GSL_HAFSV0.A_13km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-62.0 ESGgrid_LAT_CTR=22.0 @@ -984,7 +984,7 @@ case ${PREDEF_GRID_NAME} in # "GSL_HAFSV0.A_3km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-62.0 ESGgrid_LAT_CTR=22.0 @@ -1026,7 +1026,7 @@ case ${PREDEF_GRID_NAME} in # "GSD_HRRR_AK_50km") - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-163.5 ESGgrid_LAT_CTR=62.8 @@ -1077,16 +1077,16 @@ case ${PREDEF_GRID_NAME} in # GFDLgrid_STRETCH_FAC=0.63 # GFDLgrid_RES="384" # GFDLgrid_REFINE_RATIO=3 -# +# # num_margin_cells_T6_left=10 # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) -# +# # num_margin_cells_T6_right=10 # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) -# +# # num_margin_cells_T6_bottom=10 # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) -# +# # num_margin_cells_T6_top=10 # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # @@ -1114,7 +1114,7 @@ case ${PREDEF_GRID_NAME} in # # elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - GRID_GEN_METHOD= "ESGgrid" + GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR=-106.0 ESGgrid_LAT_CTR=54.0 diff --git a/ush/setup.sh b/ush/setup.sh index 112dde1cc2..605fc24026 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1901,63 +1901,6 @@ component if it is being used) are: # #----------------------------------------------------------------------- # -# Make sure that the number of cells in the x and y direction are divi- -# sible by the MPI task dimensions LAYOUT_X and LAYOUT_Y, respectively. -# -#----------------------------------------------------------------------- -# -rem=$(( NX%LAYOUT_X )) -if [ $rem -ne 0 ]; then - print_err_msg_exit "\ -The number of grid cells in the x direction (NX) is not evenly divisible -by the number of MPI tasks in the x direction (LAYOUT_X): - NX = $NX - LAYOUT_X = ${LAYOUT_X}" -fi - -rem=$(( NY%LAYOUT_Y )) -if [ $rem -ne 0 ]; then - print_err_msg_exit "\ -The number of grid cells in the y direction (NY) is not evenly divisible -by the number of MPI tasks in the y direction (LAYOUT_Y): - NY = $NY - LAYOUT_Y = ${LAYOUT_Y}" -fi - -print_info_msg "$VERBOSE" " -The MPI task layout is: - LAYOUT_X = ${LAYOUT_X} - LAYOUT_Y = ${LAYOUT_Y}" -# -#----------------------------------------------------------------------- -# -# Make sure that, for a given MPI task, the number columns (which is -# equal to the number of horizontal cells) is divisible by BLOCKSIZE. -# -#----------------------------------------------------------------------- -# -nx_per_task=$(( NX/LAYOUT_X )) -ny_per_task=$(( NY/LAYOUT_Y )) -num_cols_per_task=$(( $nx_per_task*$ny_per_task )) - -rem=$(( num_cols_per_task%BLOCKSIZE )) -if [ $rem -ne 0 ]; then - prime_factors_num_cols_per_task=$( factor $num_cols_per_task | sed -r -e 's/^[0-9]+: (.*)/\1/' ) - print_err_msg_exit "\ -The number of columns assigned to a given MPI task must be divisible by -BLOCKSIZE: - nx_per_task = NX/LAYOUT_X = $NX/${LAYOUT_X} = ${nx_per_task} - ny_per_task = NY/LAYOUT_Y = $NY/${LAYOUT_Y} = ${ny_per_task} - num_cols_per_task = nx_per_task*ny_per_task = ${num_cols_per_task} - BLOCKSIZE = $BLOCKSIZE - rem = num_cols_per_task%%BLOCKSIZE = $rem -The prime factors of num_cols_per_task are (useful for determining a va- -lid BLOCKSIZE): - prime_factors_num_cols_per_task: ${prime_factors_num_cols_per_task}" -fi -# -#----------------------------------------------------------------------- -# # If the write-component is going to be used to write output files to # disk (i.e. if QUILTING is set to "TRUE"), make sure that the grid type # used by the write-component (WRTCMP_output_grid) is set to a valid value. @@ -1975,35 +1918,6 @@ fi # #----------------------------------------------------------------------- # -# If the write component is going to be used, make sure that the number -# of grid cells in the y direction (NY) is divisible by the number of -# write tasks per group. This is because the NY rows of the grid must -# be distributed evenly among the write_tasks_per_group tasks in a given -# write group, i.e. each task must receive the same number of rows. -# This implies that NY must be evenly divisible by WRTCMP_write_tasks_- -# per_group. If it isn't, the write component will hang or fail. We -# check for this below. -# -#----------------------------------------------------------------------- -# -if [ "$QUILTING" = "TRUE" ]; then - - rem=$(( NY%WRTCMP_write_tasks_per_group )) - - if [ $rem -ne 0 ]; then - print_err_msg_exit "\ -The number of grid points in the y direction on the regional grid (ny_- -T7) must be evenly divisible by the number of tasks per write group -(WRTCMP_write_tasks_per_group): - NY = $NY - WRTCMP_write_tasks_per_group = ${WRTCMP_write_tasks_per_group} - NY%%write_tasks_per_group = $rem" - fi - -fi -# -#----------------------------------------------------------------------- -# # Calculate the number of nodes (NNODES_RUN_FCST) to request from the job # scheduler for the forecast task (RUN_FCST_TN). This is just PE_MEMBER01 # dividied by the number of processes per node we want to request for this diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index db2d10d90a..9f287aa78f 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -91,8 +91,6 @@ FV3_GFS_2017_gfdlmp: do_deep: False do_mynnedmf: !!python/none do_mynnsfclay: !!python/none - lndp_type: !!python/none - n_var_lndp: !!python/none effr_in: True fhcyc: 0.0 fhlwr: 3600.0 @@ -104,10 +102,12 @@ FV3_GFS_2017_gfdlmp: imp_physics: 11 lgfdlmprad: True lheatstrg: !!python/none - ltaerosol: !!python/none + lndp_type: !!python/none lsm: !!python/none lsoil: !!python/none lsoil_lsm: !!python/none + ltaerosol: !!python/none + n_var_lndp: !!python/none oz_phys: True oz_phys_2015: False satmedmf: !!python/none @@ -172,8 +172,6 @@ FV3_GFS_2017_gfdlmp_regional: gfs_physics_nml: <<: *gfs_2017_gfdlmp_phys cplflx: False - lndp_type: 0 - n_var_lndp: 0 iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -188,7 +186,9 @@ FV3_GFS_2017_gfdlmp_regional: iopt_tbot: 2 lgfdlmprad: True lheatstrg: False + lndp_type: 0 lsm: 1 + n_var_lndp: 0 nstf_name: [2, 0, 0, 0, 0] oz_phys: False oz_phys_2015: True @@ -327,8 +327,6 @@ FV3_GFS_v16beta: cdmbgwd: [4.0, 0.15, 1.0, 1.0] do_myjpbl: !!python/none do_myjsfc: !!python/none - lndp_type: !!python/none - n_var_lndp: !!python/none do_tofd: True do_ysu: !!python/none effr_in: True @@ -340,7 +338,9 @@ FV3_GFS_v16beta: isatmedmf: 1 lgfdlmprad: True lheatstrg: True + lndp_type: !!python/none lsoil: 4 + n_var_lndp: !!python/none nstf_name: [2, 1, 0, 0, 0] prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] @@ -395,8 +395,6 @@ FV3_CPT_v0: do_sb_physics: True do_mynnedmf: !!python/none do_mynnsfclay: !!python/none - lndp_type: !!python/none - n_var_lndp: !!python/none do_shoc: False effr_in: True fhcyc: 0.0 @@ -422,6 +420,7 @@ FV3_CPT_v0: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 + lndp_type: !!python/none lradar: False lsm: 1 lsoil: !!python/none @@ -436,6 +435,7 @@ FV3_CPT_v0: mg_do_liq_liu: True mg_qcvar: 1.0 mg_ts_auto_ice: [180.0, 900.0] + n_var_lndp: !!python/none ncld: 2 pdfflag: 4 random_clds: True diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index f6890ae008..5a02c71aff 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -139,8 +139,6 @@ debug = .false. do_mynnedmf = .true. do_mynnsfclay = .false. - lndp_type = 0 - n_var_lndp = 0 do_shum = .false. do_skeb = .false. do_sppt = .false. @@ -169,11 +167,13 @@ ivegsrc = 1 ldiag3d = .false. lheatstrg = .false. + lndp_type = 0 lradar = .true. lsm = 3 lsoil_lsm = 9 ltaerosol = .true. lwhtr = .true. + n_var_lndp = 0 ncld = 5 nst_anl = .true. nstf_name = 2,1,1,0,5 From 458d6ccb35c51360c5a9b03e522c64b1d3596ef9 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 28 Dec 2020 17:05:27 -0700 Subject: [PATCH 004/203] Automate creation of symlinks to fixed files (i.e. pregenerated grid, orography, and surface climatology files) in NCO mode; treat fixed files in NCO mode more in line with that in community mode ## DESCRIPTION OF CHANGES: ### Main changes: * In NCO mode, remove requirement that there be "fix_am" and "fix_lam" symlinks under a directory named "fix" under "regional_workflow". Instead, move "fix_am" and "fix_lam" to under the experiment directory, and have "fix_am" be a symlink like before but (for consistency with what is done in community mode) have "fix_lam" be an actual directory (not a symlink) that contains symlinks to files in some other specified directory. * For clarity, move creation of the "fix_am" symlink in NCO mode from setup.sh to generate_FV3LAM_wflow.sh, and make it part of the same if-statement in generate_FV3LAM_wflow.sh that creates the "fix_am" directory in community mode. * Assume that in NCO mode, the pregenerated grid, orography, and surface climatology files are located in the directory ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME}, where FIXLAM_NCO_BASEDIR is a new workflow variable that specifies the base directory in which these files are located. Thus, the files must be located in a subdirectory having the name of the predefined grid under ${FIXLAM_NCO_BASEDIR}. Note that FIXLAM_NCO_BASEDIR is only used in NCO mode. * With the above changes, the creation of symlinks in the "fix_lam" subdirectory under the experiment directory and the setting of the variable RES_IN_FIXLAM_FILENAMES can now be handled in the same way in NCO mode as in community mode. Thus, remove section of code in setup.sh that sets RES_IN_FIXLAM_FILENAMES specifically in NCO mode, and use the section of code that was previously only for community mode to create the symlinks and set RES_IN_FIXLAM_FILENAMES in both NCO and community modes. * Remove check near the top of setup.sh on existence of SFC_CLIMO_DIR when RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE" because this test is in effect now performed later in the script, both for NCO mode and community mode. * For simplicity, remove setting of SFC_CLIMO_DIR to a null string when it is not already a null string and RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE" because this is not done for GRID_DIR and OROG_DIR. * For the purposes of running the WE2E tests, assume that when using pregenerated grid, orography, and surface climatology files in community mode, the files are located directly under the ${PREDEF_GRID_NAME} subdirectory of some specified base directory just as we assume is the case in NCO mode. Thus, in run_experiments.sh, remove the "grid", "orog", and "sfc_climo" subdirectories in the settings of GRID_DIR, OROG_DIR, and SFC_CLIMO_DIR. Note that this is only for the WE2E testing system. Users can still specify (in config.sh) independent directories for GRID_DIR, OROG_DIR, and SFC_CLIMO_DIR when using pregenerated files in community mode. * In the sample NCO mode workflow configuration file config.nco.sh, added the variable FIXLAM_NCO_BASEDIR (for a valid value on Hera only; values are platform and user dependent). ### Improvements: * Add missing code that checks that RUN_TASK_MAKE_OROG is set to a valid value (just like there is code to check that RUN_TASK_MAKE_GRID and RUN_TASK_MAKE_SFC_CLIMO are set to valid values). * Edit error messages in setup.sh to make them more accurate and clearer. * Clean up comments. ## TESTS CONDUCTED: Ran the following WE2E tests on Hera: * nco_CONUS_25km_GFDLgrid * nco_RRFS_CONUS_25km_HRRR_RAP * pregen_grid_orog_sfc_climo * regional_002 These are ones that test the workflow in NCO mode (nco_CONUS_25km_GFDLgrid, nco_RRFS_CONUS_25km_HRRR_RAP), in community mode with pregenerated files (pregen_grid_orog_sfc_climo), and in community mode without pregenerated files (regional_002). All tests were successful. --- jobs/JREGIONAL_MAKE_GRID | 6 +- scripts/exregional_make_orog.sh | 1 - tests/run_experiments.sh | 76 ++---- ush/config.nco.sh | 18 +- ush/config_defaults.sh | 14 ++ ush/generate_FV3LAM_wflow.sh | 35 ++- ush/setup.sh | 414 ++++++++++++++------------------ 7 files changed, 257 insertions(+), 307 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index 0ff122bad1..641196137b 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -13,8 +13,8 @@ # 1) grid_gen_scr: # # This script generates grid files that will be used by subsequent -# preprocessing steps. It places its output in the temporary direc- -# tory defined in GRID_DIR. Note that: +# preprocessing steps. It places its output in the directory defined +# by GRID_DIR. Note that: # # a) This script creates grid files for each of the 7 tiles of the # cubed sphere grid (where tiles 1 through 6 cover the globe, and @@ -44,7 +44,7 @@ # 2) orog_gen_scr: # # This script generates the orography file. It places its output in -# the temporary directory defined in OROG_DIR. Note that: +# the directory defined by OROG_DIR. Note that: # # a) This script generates an orography file only on tile 7. # diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index bc817088df..2d4cdd0b56 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -350,7 +350,6 @@ fn_suffix_with_halo="tile${TILE_RGNL}.halo${NHW}.nc" raw_orog_fn="${raw_orog_fn_prefix}.${fn_suffix_with_halo}" raw_orog_fp="${raw_dir}/${raw_orog_fn}" mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}" - # #----------------------------------------------------------------------- # diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 90539bb42d..46264f1e6c 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -493,6 +493,13 @@ VERBOSE=\"${VERBOSE}\"" [ ${RUN_TASK_MAKE_OROG} = "FALSE" ] || \ [ ${RUN_TASK_MAKE_SFC_CLIMO} = "FALSE" ]; then +# Note: +# Now that the "grid", "orog", and "sfc_climo" sub-subdirectories under +# pregen_basedir have been removed, we don't need the variable pregen_basedir +# and can instead have the variable "pregen_dir" that gets set to +# ${pregen_basedir}/${PREDEF_GRID_NAME}, and pregen_dir can then be used +# to set GRID_DIR, OROG_DIR, and/or SFC_CLIMO_DIR below. + if [ "$MACHINE" = "HERA" ]; then pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "CHEYENNE" ]; then @@ -510,19 +517,7 @@ specified for this machine (MACHINE): # Directory for pregenerated grid files. # if [ ${RUN_TASK_MAKE_GRID} = "FALSE" ]; then - - GRID_DIR="${pregen_basedir}/grid/${PREDEF_GRID_NAME}" - if [ "$MACHINE" = "HERA" ]; then - GRID_DIR="/scratch2/BMC/det/FV3LAM_pregen/grid/${PREDEF_GRID_NAME}" - elif [ "$MACHINE" = "CHEYENNE" ]; then - GRID_DIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen/grid/${PREDEF_GRID_NAME}" - else - print_err_msg_exit "\ -The directory (GRID_DIR) in which the pregenerated grid files are located -has not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - + GRID_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" str=${str}" # # Directory containing the pregenerated grid files. @@ -534,18 +529,7 @@ GRID_DIR=\"${GRID_DIR}\"" # Directory for pregenerated orography files. # if [ ${RUN_TASK_MAKE_OROG} = "FALSE" ]; then - - if [ "$MACHINE" = "HERA" ]; then - OROG_DIR="/scratch2/BMC/det/FV3LAM_pregen/orog/${PREDEF_GRID_NAME}" - elif [ "$MACHINE" = "CHEYENNE" ]; then - OROG_DIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen/orog/${PREDEF_GRID_NAME}" - else - print_err_msg_exit "\ -The directory (OROG_DIR) in which the pregenerated grid files are located -has not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - + OROG_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" str=${str}" # # Directory containing the pregenerated orography files. @@ -557,18 +541,7 @@ OROG_DIR=\"${OROG_DIR}\"" # Directory for pregenerated surface climatology files. # if [ ${RUN_TASK_MAKE_SFC_CLIMO} = "FALSE" ]; then - - if [ "$MACHINE" = "HERA" ]; then - SFC_CLIMO_DIR="/scratch2/BMC/det/FV3LAM_pregen/sfc_climo/${PREDEF_GRID_NAME}" - elif [ "$MACHINE" = "CHEYENNE" ]; then - SFC_CLIMO_DIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen/sfc_climo/${PREDEF_GRID_NAME}" - else - print_err_msg_exit "\ -The directory (SFC_CLIMO_DIR) in which the pregenerated grid files are -located has not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - + SFC_CLIMO_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" str=${str}" # # Directory containing the pregenerated surface climatology files. @@ -628,21 +601,7 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # \$PTMP/com/\$NET/\$RUN/\$RUN.\$yyyymmdd/\$hh # RUN=\"\${EXPT_SUBDIR}\" -envir=\"\${EXPT_SUBDIR}\" -# -# In NCO mode, the user must manually (e.g. after doing the build step) -# create the symlink \"\${FIXrrfs}/fix_sar\" that points to EMC's FIXsar -# directory on the machine. For example, on hera, the symlink's target -# needs to be -# -# /scratch2/NCEPDEV/fv3-cam/emc.campara/fix_fv3cam/fix_sar -# -# The experiment generation script will then set FIXsar to -# -# FIXsar=\"\${FIXrrfs}/fix_sar/\${PREDEF_GRID_NAME}\" -# -# where PREDEF_GRID_NAME has the value set above. -#" +envir=\"\${EXPT_SUBDIR}\"" # # Set COMINgfs. # @@ -653,10 +612,13 @@ envir=\"\${EXPT_SUBDIR}\" if [ "$MACHINE" = "HERA" ]; then COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" + FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" + FIXLAM_NCO_BASEDIR="/needs/to/be/specified" elif [ "$MACHINE" = "CHEYENNE" ]; then COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" + FIXLAM_NCO_BASEDIR="/needs/to/be/specified" else print_err_msg_exit "\ The directory (COMINgfs) that needs to be specified when running the @@ -672,7 +634,15 @@ for this machine (MACHINE): # mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as # the external model for ICs and/or LBCs. # -COMINgfs=\"${COMINgfs}\"" +COMINgfs=\"${COMINgfs}\" +# +# The base directory in which the pregenerated grid, orography, and surface +# climatology \"fixed\" files used in NCO mode are located. In NCO mode, +# the workflow scripts will create a symlink (at the location specified +# by FIXLAM) to a subdirectory under FIXLAM_NCO_BASDEDIR. (The name of +# the subdirectory is the name of the grid specified by PREDEF_GRID_NAME.) +# +FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" fi # diff --git a/ush/config.nco.sh b/ush/config.nco.sh index 84c381e021..58a4c01003 100644 --- a/ush/config.nco.sh +++ b/ush/config.nco.sh @@ -22,24 +22,12 @@ CYCL_HRS=( "18" ) EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" +# # The following must be modified for different platforms and users. +# RUN="an_experiment" COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" # Path to directory containing files from the external model (FV3GFS). +FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" # Path to directory containing the pregenerated grid, orography, and surface climatology "fixed" files to use for the experiment. STMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/stmp" # Path to directory STMP that mostly contains input files. PTMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/ptmp" # Path to directory PTMP in which the experiment's output files will be placed. -# -# In NCO mode, the user must manually (e.g. after doing the build step) -# create the symlink "${FIXrrfs}/fix_sar" that points to EMC's FIXLAM -# directory on the machine. For example, on hera, the symlink's target -# needs to be -# -# /scratch2/NCEPDEV/fv3-cam/emc.campara/fix_fv3cam/fix_sar -# -# The experiment generation script will then set FIXLAM to -# -# FIXLAM="${FIXrrfs}/fix_lam/${PREDEF_GRID_NAME}" -# -# where PREDEF_GRID_NAME has the value set above. -# - diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index c9f284eb7f..cd48dc2dc7 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -175,6 +175,19 @@ EXPT_SUBDIR="" # # $COMINgfs/gfs.$yyyymmdd/$hh # +# FIXLAM_NCO_BASEDIR: +# The base directory containing pregenerated grid, orography, and surface +# climatology files. For the pregenerated grid specified by PREDEF_GRID_NAME, +# these "fixed" files are located in: +# +# ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME} +# +# The workflow scripts will create a symlink in the experiment directory +# that will point to a subdirectory (having the name of the grid being +# used) under this directory. This variable should be set to a null +# string in this file, but it can be specified in the user-specified +# workflow configuration file (EXPT_CONFIG_FN) +# # STMP: # The beginning portion of the directory that will contain cycle-dependent # model input files, symlinks to cycle-independent input files, and raw @@ -214,6 +227,7 @@ EXPT_SUBDIR="" #----------------------------------------------------------------------- # COMINgfs="/base/path/of/directory/containing/gfs/input/files" +FIXLAM_NCO_BASEDIR="" STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" NET="rrfs" envir="para" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 48453ac232..6701c25c20 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -407,16 +407,39 @@ fi # #----------------------------------------------------------------------- # -# Copy fixed files from system directory to the FIXam directory (which -# is under the experiment directory). Note that some of these files get -# renamed during the copy process. +# Create the FIXam directory under the experiment directory. In NCO mode, +# this will be a symlink to the directory specified in FIXgsm, while in +# community mode, it will be an actual directory with files copied into +# it from FIXgsm. # #----------------------------------------------------------------------- # +# First, consider NCO mode. +# +if [ "${RUN_ENVIR}" = "nco" ]; then -# In NCO mode, we assume the following copy operation is done beforehand, -# but that can be changed. -if [ "${RUN_ENVIR}" != "nco" ]; then + ln_vrfy -fsn "$FIXgsm" "$FIXam" +# +# Resolve the target directory that the FIXam symlink points to and check +# that it exists. +# + path_resolved=$( readlink -m "$FIXam" ) + if [ ! -d "${path_resolved}" ]; then + print_err_msg_exit "\ +In order to be able to generate a forecast experiment in NCO mode (i.e. +when RUN_ENVIR set to \"nco\"), the path specified by FIXam after resolving +all symlinks (path_resolved) must be an existing directory (but in this +case isn't): + RUN_ENVIR = \"${RUN_ENVIR}\" + FIXam = \"$FIXam\" + path_resolved = \"${path_resolved}\" +Please ensure that path_resolved is an existing directory and then rerun +the experiment generation script." + fi +# +# Now consider community mode. +# +else print_info_msg "$VERBOSE" " Copying fixed files from system directory (FIXgsm) to a subdirectory diff --git a/ush/setup.sh b/ush/setup.sh index 605fc24026..dcd2a4fe01 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -190,6 +190,26 @@ fi # #----------------------------------------------------------------------- # +# Make sure that RUN_TASK_MAKE_OROG is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_RUN_TASK_MAKE_OROG" +# +# Set RUN_TASK_MAKE_OROG to either "TRUE" or "FALSE" so we don't have to +# consider other valid values later on. +# +RUN_TASK_MAKE_OROG=${RUN_TASK_MAKE_OROG^^} +if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ + [ "${RUN_TASK_MAKE_OROG}" = "YES" ]; then + RUN_TASK_MAKE_OROG="TRUE" +elif [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ] || \ + [ "${RUN_TASK_MAKE_OROG}" = "NO" ]; then + RUN_TASK_MAKE_OROG="FALSE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that RUN_TASK_MAKE_SFC_CLIMO is set to a valid value. # #----------------------------------------------------------------------- @@ -209,28 +229,6 @@ elif [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ] || \ RUN_TASK_MAKE_SFC_CLIMO="FALSE" fi # -# If RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE", make sure that the di- -# rectory SFC_CLIMO_DIR that should contain the pre-generated surface -# climatology files exists. -# -if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ] && \ - [ ! -d "${SFC_CLIMO_DIR}" ]; then - print_err_msg_exit "\ -The directory (SFC_CLIMO_DIR) that should contain the pre-generated sur- -face climatology files does not exist: - SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\"" -fi -# -# If RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE" and the variable specifying -# the directory in which to look for pregenerated grid and orography files -# (i.e. SFC_CLIMO_DIR) is not empty, then for clarity reset the latter to -# an empty string (because it will not be used). -# -if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] && \ - [ -n "${SFC_CLIMO_DIR}" ]; then - SFC_CLIMO_DIR="" -fi -# #----------------------------------------------------------------------- # # Make sure that DO_SHUM is set to a valid value. @@ -660,7 +658,6 @@ SRC_DIR="${SR_WX_APP_TOP_DIR}/src" PARMDIR="$HOMErrfs/parm" MODULES_DIR="$HOMErrfs/modulefiles" EXECDIR="${SR_WX_APP_TOP_DIR}/bin" -FIXrrfs="$HOMErrfs/fix" TEMPLATE_DIR="$USHDIR/templates" case $MACHINE in @@ -669,48 +666,56 @@ case $MACHINE in FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "WCOSS_DELL_P3") FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "HERA") FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} ;; "ORION") FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "JET") FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "ODIN") FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "CHEYENNE") FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; "STAMPEDE") FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} + FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; *) @@ -720,6 +725,7 @@ One or more fix file directories have not been specified for this machine: FIXgsm = \"${FIXgsm:-\"\"} TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} You can specify the missing location(s) in config.sh" ;; @@ -1011,67 +1017,19 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # LOGDIR="${EXPTDIR}/log" -if [ "${RUN_ENVIR}" = "nco" ]; then +FIXam="${EXPTDIR}/fix_am" +FIXLAM="${EXPTDIR}/fix_lam" - FIXam="${FIXrrfs}/fix_am" -# -# In NCO mode (i.e. if RUN_ENVIR set to "nco"), it is assumed that before -# running the experiment generation script, the path specified in FIXam -# already exists and is either itself the directory in which various fixed -# files (but not the ones containing the regional grid and the orography -# and surface climatology on that grid) are located, or it is a symlink -# to such a directory. Resolve any symlinks in the path specified by -# FIXam and check that this is the case. -# - path_resolved=$( readlink -m "$FIXam" ) - if [ ! -d "${path_resolved}" ]; then - print_err_msg_exit "\ -In order to be able to generate a forecast experiment in NCO mode (i.e. -when RUN_ENVIR set to \"nco\"), the path specified by FIXam after resolving -all symlinks (path_resolved) must be an existing directory (but in this -case isn't): - RUN_ENVIR = \"${RUN_ENVIR}\" - FIXam = \"$FIXam\" - path_resolved = \"${path_resolved}\" -Please ensure that path_resolved is an existing directory and then rerun -the experiment generation script." - fi - - FIXLAM="${FIXrrfs}/fix_lam/${PREDEF_GRID_NAME}" -# -# In NCO mode (i.e. if RUN_ENVIR set to "nco"), it is assumed that before -# running the experiment generation script, the path specified in FIXLAM -# already exists and is either itself the directory in which the fixed -# grid, orography, and surface climatology files are located, or it is a -# symlink to such a directory. Resolve any symlinks in the path specified -# by FIXLAM and check that this is the case. -# - path_resolved=$( readlink -m "$FIXLAM" ) - if [ ! -d "${path_resolved}" ]; then - print_err_msg_exit "\ -In order to be able to generate a forecast experiment in NCO mode (i.e. -when RUN_ENVIR set to \"nco\"), the path specified by FIXLAM after resolving -all symlinks (path_resolved) must be an existing directory (but in this -case isn't): - RUN_ENVIR = \"${RUN_ENVIR}\" - FIXLAM = \"$FIXLAM\" - path_resolved = \"${path_resolved}\" -Please ensure that path_resolved is an existing directory and then rerun -the experiment generation script." - fi +if [ "${RUN_ENVIR}" = "nco" ]; then CYCLE_BASEDIR="$STMP/tmpnwprd/$RUN" check_for_preexist_dir_file "${CYCLE_BASEDIR}" "${PREEXISTING_DIR_METHOD}" - COMROOT="$PTMP/com" - COMOUT_BASEDIR="$COMROOT/$NET/$envir" check_for_preexist_dir_file "${COMOUT_BASEDIR}" "${PREEXISTING_DIR_METHOD}" else - FIXam="${EXPTDIR}/fix_am" - FIXLAM="${EXPTDIR}/fix_lam" CYCLE_BASEDIR="$EXPTDIR" COMROOT="" COMOUT_BASEDIR="" @@ -1352,28 +1310,49 @@ LOAD_MODULES_RUN_TASK_FP="$USHDIR/load_modules_run_task.sh" # RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE") or searched for (if RUN_- # TASK_MAKE_SFC_CLIMO is set to "FALSE"). # +# First, consider NCO mode. +# #---------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then + nco_fix_dir="${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME}" + if [ ! -d "${nco_fix_dir}" ]; then + print_err_msg_exit "\ +The directory (nco_fix_dir) that should contain the pregenerated grid, +orography, and surface climatology files does not exist: + nco_fix_dir = \"${nco_fix_dir}\"" + fi + if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_GRID}" = "FALSE" -a \ - "${GRID_DIR}" != "$FIXLAM" ]; then + "${GRID_DIR}" != "${nco_fix_dir}" ]; then msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that grid files already -exist in the directory specified by FIXLAM. Thus, the grid file genera- -tion task must not be run (i.e. RUN_TASK_MAKE_GRID must be set to -FALSE), and the directory in which to look for the grid files (i.e. -GRID_DIR) must be set to FIXLAM. Current values for these quantities -are: +When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated +grid files already exist in the directory + + \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + +where + + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" + +Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must +be set to \"FALSE\"), and the directory in which to look for the grid +files (i.e. GRID_DIR) must be set to the one above. Current values for +these quantities are: + RUN_TASK_MAKE_GRID = \"${RUN_TASK_MAKE_GRID}\" GRID_DIR = \"${GRID_DIR}\" -Resetting RUN_TASK_MAKE_GRID to \"FALSE\" and GRID_DIR to the contents -of FIXLAM. Reset values are:" + +Resetting RUN_TASK_MAKE_GRID to \"FALSE\" and GRID_DIR to the one above. +Reset values are: +" RUN_TASK_MAKE_GRID="FALSE" - GRID_DIR="$FIXLAM" + GRID_DIR="${nco_fix_dir}" msg="$msg"" RUN_TASK_MAKE_GRID = \"${RUN_TASK_MAKE_GRID}\" @@ -1381,27 +1360,38 @@ of FIXLAM. Reset values are:" " print_info_msg "$msg" - + fi if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_OROG}" = "FALSE" -a \ - "${OROG_DIR}" != "$FIXLAM" ]; then + "${OROG_DIR}" != "${nco_fix_dir}" ]; then msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that orography files al- -ready exist in the directory specified by FIXLAM. Thus, the orography -file generation task must not be run (i.e. RUN_TASK_MAKE_OROG must be -set to FALSE), and the directory in which to look for the orography -files (i.e. OROG_DIR) must be set to FIXLAM. Current values for these -quantities are: +When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated +orography files already exist in the directory + + \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + +where + + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" + +Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must +be set to \"FALSE\"), and the directory in which to look for the orography +files (i.e. OROG_DIR) must be set to the one above. Current values for +these quantities are: + RUN_TASK_MAKE_OROG = \"${RUN_TASK_MAKE_OROG}\" OROG_DIR = \"${OROG_DIR}\" -Resetting RUN_TASK_MAKE_OROG to \"FALSE\" and OROG_DIR to the contents -of FIXLAM. Reset values are:" + +Resetting RUN_TASK_MAKE_OROG to \"FALSE\" and OROG_DIR to the one above. +Reset values are: +" RUN_TASK_MAKE_OROG="FALSE" - OROG_DIR="$FIXLAM" + OROG_DIR="${nco_fix_dir}" msg="$msg"" RUN_TASK_MAKE_OROG = \"${RUN_TASK_MAKE_OROG}\" @@ -1409,27 +1399,38 @@ of FIXLAM. Reset values are:" " print_info_msg "$msg" - + fi if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" -a \ - "${SFC_CLIMO_DIR}" != "$FIXLAM" ]; then + "${SFC_CLIMO_DIR}" != "${nco_fix_dir}" ]; then msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that surface climatology -files already exist in the directory specified by FIXLAM. Thus, the -surface climatology file generation task must not be run (i.e. RUN_- -TASK_MAKE_SFC_CLIMO must be set to FALSE), and the directory in which to -look for the surface climatology files (i.e. SFC_CLIMO_DIR) must be set -to FIXLAM. Current values for these quantities are: +When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated +surface climatology files already exist in the directory + + \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + +where + + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" + +Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO +must be set to \"FALSE\"), and the directory in which to look for the +surface climatology files (i.e. SFC_CLIMO_DIR) must be set to the one +above. Current values for these quantities are: + RUN_TASK_MAKE_SFC_CLIMO = \"${RUN_TASK_MAKE_SFC_CLIMO}\" SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\" -Resetting RUN_TASK_MAKE_SFC_CLIMO to \"FALSE\" and SFC_CLIMO_DIR to the -contents of FIXLAM. Reset values are:" + +Resetting RUN_TASK_MAKE_SFC_CLIMO to \"FALSE\" and SFC_CLIMO_DIR to the +one above. Reset values are: +" RUN_TASK_MAKE_SFC_CLIMO="FALSE" - SFC_CLIMO_DIR="$FIXLAM" + SFC_CLIMO_DIR="${nco_fix_dir}" msg="$msg"" RUN_TASK_MAKE_SFC_CLIMO = \"${RUN_TASK_MAKE_SFC_CLIMO}\" @@ -1437,44 +1438,42 @@ contents of FIXLAM. Reset values are:" " print_info_msg "$msg" - - fi -else + fi # #----------------------------------------------------------------------- # -# If RUN_TASK_MAKE_GRID is set to "FALSE", the workflow will look for -# the pre-generated grid files in GRID_DIR. In this case, make sure -# that GRID_DIR exists. Otherwise, set it to a predefined location un- -# der the experiment directory (EXPTDIR). +# Now consider community mode. # #----------------------------------------------------------------------- +# +else +# +# If RUN_TASK_MAKE_GRID is set to "FALSE", the workflow will look for +# the pregenerated grid files in GRID_DIR. In this case, make sure that +# GRID_DIR exists. Otherwise, set it to a predefined location under the +# experiment directory (EXPTDIR). # if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then if [ ! -d "${GRID_DIR}" ]; then print_err_msg_exit "\ -The directory (GRID_DIR) that should contain the pre-generated grid -files does not exist: +The directory (GRID_DIR) that should contain the pregenerated grid files +does not exist: GRID_DIR = \"${GRID_DIR}\"" fi else GRID_DIR="$EXPTDIR/grid" fi # -#----------------------------------------------------------------------- -# # If RUN_TASK_MAKE_OROG is set to "FALSE", the workflow will look for -# the pre-generated orography files in OROG_DIR. In this case, make -# sure that OROG_DIR exists. Otherwise, set it to a predefined location -# under the experiment directory (EXPTDIR). -# -#----------------------------------------------------------------------- +# the pregenerated orography files in OROG_DIR. In this case, make sure +# that OROG_DIR exists. Otherwise, set it to a predefined location under +# the experiment directory (EXPTDIR). # if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then if [ ! -d "${OROG_DIR}" ]; then print_err_msg_exit "\ -The directory (OROG_DIR) that should contain the pre-generated orography +The directory (OROG_DIR) that should contain the pregenerated orography files does not exist: OROG_DIR = \"${OROG_DIR}\"" fi @@ -1482,24 +1481,18 @@ files does not exist: OROG_DIR="$EXPTDIR/orog" fi # -#----------------------------------------------------------------------- -# # If RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE", the workflow will look -# for the pre-generated surface climatology files in SFC_CLIMO_DIR. In +# for the pregenerated surface climatology files in SFC_CLIMO_DIR. In # this case, make sure that SFC_CLIMO_DIR exists. Otherwise, set it to # a predefined location under the experiment directory (EXPTDIR). -# -#----------------------------------------------------------------------- # if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - if [ ! -d "${SFC_CLIMO_DIR}" ]; then print_err_msg_exit "\ -The directory (SFC_CLIMO_DIR) that should contain the pre-generated orography -files does not exist: +The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface +climatology files does not exist: SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\"" fi - else SFC_CLIMO_DIR="$EXPTDIR/sfc_climo" fi @@ -1688,51 +1681,28 @@ fi # #----------------------------------------------------------------------- # -# +# Create a new experiment directory. Note that at this point we are +# guaranteed that there is no preexisting experiment directory. # #----------------------------------------------------------------------- # -RES_IN_FIXLAM_FILENAMES="" +mkdir_vrfy -p "$EXPTDIR" + -if [ "${RUN_ENVIR}" != "nco" ]; then - mkdir_vrfy -p "$FIXLAM" -fi # #----------------------------------------------------------------------- # -# +# If not running the MAKE_GRID_TN, MAKE_OROG_TN, and/or MAKE_SFC_CLIMO +# tasks, create symlinks under the FIXLAM directory to pregenerated grid, +# orography, and surface climatology files. In the process, also set +# RES_IN_FIXLAM_FILENAMES, which is the resolution of the grid (in units +# of number of grid points on an equivalent global uniform cubed-sphere +# grid) used in the names of the fixed files in the FIXLAM directory. # #----------------------------------------------------------------------- # -if [ "${RUN_ENVIR}" = "nco" ]; then - - suffix="${DOT_OR_USCORE}mosaic.halo${NH3}.nc" - glob_pattern="C*$suffix" - cd_vrfy $FIXLAM - num_files=$( ls -1 ${glob_pattern} 2>/dev/null | wc -l ) - - if [ "${num_files}" -ne "1" ]; then - print_err_msg_exit "\ -Exactly one file must exist in directory FIXLAM matching the globbing -pattern glob_pattern: - FIXLAM = \"${FIXLAM}\" - glob_pattern = \"${glob_pattern}\" - num_files = ${num_files}" - fi - - fn=$( ls -1 ${glob_pattern} ) - RES_IN_FIXLAM_FILENAMES=$( \ - printf "%s" $fn | sed -n -r -e "s/^C([0-9]*)$suffix/\1/p" ) - if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ] && \ - [ "${GFDLgrid_RES}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then - print_err_msg_exit "\ -The resolution extracted from the fixed file names (RES_IN_FIXLAM_FILENAMES) -does not match the resolution specified by GFDLgrid_RES: - GFDLgrid_RES = ${GFDLgrid_RES} - RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" - fi - -else +mkdir_vrfy -p "$FIXLAM" +RES_IN_FIXLAM_FILENAMES="" # #----------------------------------------------------------------------- # @@ -1742,19 +1712,19 @@ else # #----------------------------------------------------------------------- # - res_in_grid_fns="" - if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then +res_in_grid_fns="" +if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - link_fix \ - verbose="$VERBOSE" \ - file_group="grid" \ - output_varname_res_in_filenames="res_in_grid_fns" || \ - print_err_msg_exit "\ - Call to function to create links to grid files failed." + link_fix \ + verbose="$VERBOSE" \ + file_group="grid" \ + output_varname_res_in_filenames="res_in_grid_fns" || \ + print_err_msg_exit "\ +Call to function to create links to grid files failed." - RES_IN_FIXLAM_FILENAMES="${res_in_grid_fns}" + RES_IN_FIXLAM_FILENAMES="${res_in_grid_fns}" - fi +fi # #----------------------------------------------------------------------- # @@ -1764,29 +1734,29 @@ else # #----------------------------------------------------------------------- # - res_in_orog_fns="" - if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then +res_in_orog_fns="" +if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then - link_fix \ - verbose="$VERBOSE" \ - file_group="orog" \ - output_varname_res_in_filenames="res_in_orog_fns" || \ - print_err_msg_exit "\ - Call to function to create links to orography files failed." - - if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ - [ "${res_in_orog_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then - print_err_msg_exit "\ - The resolution extracted from the orography file names (res_in_orog_fns) - does not match the resolution in other groups of files already consi- - dered (RES_IN_FIXLAM_FILENAMES): - res_in_orog_fns = ${res_in_orog_fns} - RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" - else - RES_IN_FIXLAM_FILENAMES="${res_in_orog_fns}" - fi + link_fix \ + verbose="$VERBOSE" \ + file_group="orog" \ + output_varname_res_in_filenames="res_in_orog_fns" || \ + print_err_msg_exit "\ +Call to function to create links to orography files failed." + if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ + [ "${res_in_orog_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then + print_err_msg_exit "\ +The resolution extracted from the orography file names (res_in_orog_fns) +does not match the resolution in other groups of files already consi- +dered (RES_IN_FIXLAM_FILENAMES): + res_in_orog_fns = ${res_in_orog_fns} + RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" + else + RES_IN_FIXLAM_FILENAMES="${res_in_orog_fns}" fi + +fi # #----------------------------------------------------------------------- # @@ -1797,28 +1767,26 @@ else # #----------------------------------------------------------------------- # - res_in_sfc_climo_fns="" - if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - - link_fix \ - verbose="$VERBOSE" \ - file_group="sfc_climo" \ - output_varname_res_in_filenames="res_in_sfc_climo_fns" || \ - print_err_msg_exit "\ - Call to function to create links to surface climatology files failed." +res_in_sfc_climo_fns="" +if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ - [ "${res_in_sfc_climo_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then - print_err_msg_exit "\ - The resolution extracted from the surface climatology file names (res_- - in_sfc_climo_fns) does not match the resolution in other groups of files - already considered (RES_IN_FIXLAM_FILENAMES): - res_in_sfc_climo_fns = ${res_in_sfc_climo_fns} - RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" - else - RES_IN_FIXLAM_FILENAMES="${res_in_sfc_climo_fns}" - fi + link_fix \ + verbose="$VERBOSE" \ + file_group="sfc_climo" \ + output_varname_res_in_filenames="res_in_sfc_climo_fns" || \ + print_err_msg_exit "\ +Call to function to create links to surface climatology files failed." + if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ + [ "${res_in_sfc_climo_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then + print_err_msg_exit "\ +The resolution extracted from the surface climatology file names (res_- +in_sfc_climo_fns) does not match the resolution in other groups of files +already considered (RES_IN_FIXLAM_FILENAMES): + res_in_sfc_climo_fns = ${res_in_sfc_climo_fns} + RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" + else + RES_IN_FIXLAM_FILENAMES="${res_in_sfc_climo_fns}" fi fi @@ -1937,17 +1905,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) -# -#----------------------------------------------------------------------- -# -# Create a new experiment directory. Note that at this point we are -# guaranteed that there is no preexisting experiment directory. -# -#----------------------------------------------------------------------- -# -mkdir_vrfy -p "$EXPTDIR" - - @@ -2290,7 +2247,6 @@ SRC_DIR="$SRC_DIR" PARMDIR="$PARMDIR" MODULES_DIR="${MODULES_DIR}" EXECDIR="$EXECDIR" -FIXrrfs="$FIXrrfs" FIXam="$FIXam" FIXLAM="$FIXLAM" FIXgsm="$FIXgsm" From d5287f5fb07d6e73078da3796504ecfa118145ca Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 6 Jan 2021 11:32:48 -0700 Subject: [PATCH 005/203] Enable fetching of FV3GFS files from NOAA HPSS after 202002600 (as well as before) for both nemsio and grib2 formats (#391) ## DESCRIPTION OF CHANGES: * Modify workflow script get_extrn_mdl_dir_file_info.sh to enable fetching of FV3GFS files from NOAA HPSS on and after 202002600, which was broken due to a change in file naming convention on HPSS starting on this date. * Files before this date (and after 20190612, which is the start of operations for FV3GFS) can still be fetched as usual. * New code works for both nemsio-formatted and grib2-formatted FV3GFS files on HPSS. * Add 10 new WE2E test configurations to test this capability (both for nemsio and of grib2 formatted files) for five different dates. Dates chosen as follows: * 2019061200: Date on which FV3GFS becomes operational. Appears in if-statement in get_extrn_mdl_dir_file_info.sh. * 2020022600: Date on which there is a change in the naming convention of FV3GFS files on HPSS. Appears in if-statement in get_extrn_mdl_dir_file_info.sh. * 2020022518: Date right before the one on which there is a change in the naming convention of FV3GFS files on HPSS (i.e. 2020022600). * 2019101800: Date about halfway between the date on which FV3GFS becomes operational (2019061200) and the one on which there is the change in file naming convention (2020022600). * 2021010100: Date after the one on which there is a change in file naming convention (2020022600) and is very recent. * Rename the preexisting WE2E tests that get files from NOAA HPSS so that they use the same naming convention as the five new WE2E tests. ## TESTS CONDUCTED: On Hera, ran the 10 new WE2E tests for fetching FV3GFS files from HPSS for various dates. All succeeded. --- ...g.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh} | 0 ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh | 19 ++++ ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh | 19 ++++ ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh | 19 ++++ ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh | 19 ++++ ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh | 19 ++++ ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh | 19 ++++ ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh | 19 ++++ ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh | 19 ++++ ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh | 19 ++++ ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh | 19 ++++ ...g.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh} | 0 ...config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh} | 0 ... config.get_from_HPSS_ics_RAP_lbcs_RAP.sh} | 0 tests/baselines_list.txt | 18 +++- ush/get_extrn_mdl_file_dir_info.sh | 87 ++++++++++--------- 16 files changed, 249 insertions(+), 46 deletions(-) rename tests/baseline_configs/{config.get_extrn_files_from_hpss_FV3GFS_FV3GFS.sh => config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh} (100%) create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh rename tests/baseline_configs/{config.get_extrn_files_from_hpss_GSMGFS_GSMGFS.sh => config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh} (100%) rename tests/baseline_configs/{config.get_extrn_files_from_hpss_HRRR_RAP.sh => config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh} (100%) rename tests/baseline_configs/{config.get_extrn_files_from_hpss_RAP_RAP.sh => config.get_from_HPSS_ics_RAP_lbcs_RAP.sh} (100%) diff --git a/tests/baseline_configs/config.get_extrn_files_from_hpss_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh similarity index 100% rename from tests/baseline_configs/config.get_extrn_files_from_hpss_FV3GFS_FV3GFS.sh rename to tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh new file mode 100644 index 0000000000..6f37bdb94d --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20190612" +DATE_LAST_CYCL="20190612" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh new file mode 100644 index 0000000000..2bcdf7a5a8 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20191018" +DATE_LAST_CYCL="20191018" +CYCL_HRS=( "18" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh new file mode 100644 index 0000000000..fe2b59ae3e --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200225" +DATE_LAST_CYCL="20200225" +CYCL_HRS=( "18" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh new file mode 100644 index 0000000000..f2a33611e0 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200226" +DATE_LAST_CYCL="20200226" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh new file mode 100644 index 0000000000..09fde49fa5 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20210101" +DATE_LAST_CYCL="20210101" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh new file mode 100644 index 0000000000..ae813d595e --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20190612" +DATE_LAST_CYCL="20190612" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh new file mode 100644 index 0000000000..6a95be86c6 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20191018" +DATE_LAST_CYCL="20191018" +CYCL_HRS=( "18" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh new file mode 100644 index 0000000000..9660605f94 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200225" +DATE_LAST_CYCL="20200225" +CYCL_HRS=( "18" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh new file mode 100644 index 0000000000..c1cdc625b3 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200226" +DATE_LAST_CYCL="20200226" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh new file mode 100644 index 0000000000..431688b009 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20210101" +DATE_LAST_CYCL="20210101" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_extrn_files_from_hpss_GSMGFS_GSMGFS.sh b/tests/baseline_configs/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh similarity index 100% rename from tests/baseline_configs/config.get_extrn_files_from_hpss_GSMGFS_GSMGFS.sh rename to tests/baseline_configs/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh diff --git a/tests/baseline_configs/config.get_extrn_files_from_hpss_HRRR_RAP.sh b/tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh similarity index 100% rename from tests/baseline_configs/config.get_extrn_files_from_hpss_HRRR_RAP.sh rename to tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh diff --git a/tests/baseline_configs/config.get_extrn_files_from_hpss_RAP_RAP.sh b/tests/baseline_configs/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh similarity index 100% rename from tests/baseline_configs/config.get_extrn_files_from_hpss_RAP_RAP.sh rename to tests/baseline_configs/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index e2eb50f796..7dc4253684 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -2,10 +2,20 @@ DOT_OR_USCORE GST_release_public_v1 community_ensemble_008mems community_ensemble_2mems -get_extrn_files_from_hpss_FV3GFS_FV3GFS -get_extrn_files_from_hpss_GSMGFS_GSMGFS -get_extrn_files_from_hpss_HRRR_RAP -get_extrn_files_from_hpss_RAP_RAP +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100 +get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS +get_from_HPSS_ics_HRRR_lbcs_RAP +get_from_HPSS_ics_RAP_lbcs_RAP grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS grid_GSD_HRRR_AK_50km diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 9742b8b66e..b241dad80c 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -122,18 +122,18 @@ where the arguments are defined as follows: Name of the external model, i.e. the name of the model providing the fields from which files containing initial conditions, surface fields, and/or lateral boundary conditions for the FV3-LAM will be generated. - + anl_or_fcst: Flag that specifies whether the external model files we are interested - in obtaining are analysis or forecast files. - + in obtaining are analysis or forecast files. + cdate_FV3LAM: The cycle date and time (hours only) for which we want to obtain file and directory information. This has the form YYYYMMDDHH, where YYYY - is the four-digit starting year of the cycle, MM is the two-digit + is the four-digit starting year of the cycle, MM is the two-digit month, DD is the two-digit day of the month, and HH is the two-digit hour of day. - + time_offset_hrs: The number of hours by which to shift back in time the start time of the external model forecast from the specified cycle start time of the @@ -141,47 +141,47 @@ where the arguments are defined as follows: external model analysis files, this is normally set to 0. When get- ting directory and file information on external model forecast files, this may be set to a nonzero value to obtain information for an exter- - nal model run that started time_offset_hrs hours before cdate_FV3LAM - (instead of exactly at cdate_FV3LAM). Note that in this case, the + nal model run that started time_offset_hrs hours before cdate_FV3LAM + (instead of exactly at cdate_FV3LAM). Note that in this case, the forecast hours (relative to the external model run's start time) at which the lateral boundary conditions will be updated must be shifted forward by time_offset_hrs hours relative to those for the FV3-LAM in order to make up for the backward-in-time shift in the starting time of the external model. - + varname_extrn_mdl_cdate: - Name of the global variable that will contain the starting date and + Name of the global variable that will contain the starting date and hour of the external model run. - + varname_extrn_mdl_lbc_spec_fhrs: Name of the global variable that will contain the forecast hours (re- lative to the starting time of the external model run, which is earli- er than that of the FV3-LAM by time_offset_hrs hours) at which lateral boundary condition (LBC) output files are obtained from the external model (and will be used to update the LBCs of the FV3-LAM). - + varname_extrn_mdl_fns: Name of the global variable that will contain the names of the exter- nal model output files. - + varname_extrn_mdl_sysdir: Name of the global variable that will contain the system directory in which the externaml model output files may be stored. - + varname_extrn_mdl_arcv_fmt: Name of the global variable that will contain the format of the ar- chive file on HPSS in which the externaml model output files may be stored. - + varname_extrn_mdl_arcv_fns: Name of the global variable that will contain the name of the archive file on HPSS in which the externaml model output files may be stored. - + varname_extrn_mdl_arcv_fps: Name of the global variable that will contain the full path to the ar- - chive file on HPSS in which the externaml model output files may be + chive file on HPSS in which the externaml model output files may be stored. - + varname_extrn_mdl_arcvrel_dir: Name of the global variable that will contain the archive-relative di- rectory, i.e. the directory \"inside\" the archive file in which the ex- @@ -346,8 +346,8 @@ fi ;; "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then + + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then fns=( "atm" "sfc" ) suffix="anl.nemsio" @@ -378,7 +378,7 @@ fi fi ;; - + "RAP") # # Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter @@ -527,12 +527,12 @@ and analysis or forecast (anl_or_fcst): suffix="" fns_on_disk=( "${fns[@]/%/$suffix}" ) fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; + ;; *) print_err_msg_exit "\ The external model file names have not yet been specified for this com- -bination of external model (extrn_mdl_name) and analysis or forecast +bination of external model (extrn_mdl_name) and analysis or forecast (anl_or_fcst): extrn_mdl_name = \"${extrn_mdl_name}\" anl_or_fcst = \"${anl_or_fcst}\"" @@ -545,10 +545,10 @@ bination of external model (extrn_mdl_name) and analysis or forecast # #----------------------------------------------------------------------- # -# Set the system directory (i.e. a directory on disk) in which the external -# model output files for the specified cycle date (cdate) may be located. -# Note that this will be used by the calling script only if the output -# files for the specified cdate actually exist at this location. Otherwise, +# Set the system directory (i.e. a directory on disk) in which the external +# model output files for the specified cycle date (cdate) may be located. +# Note that this will be used by the calling script only if the output +# files for the specified cdate actually exist at this location. Otherwise, # the files will be searched for on the mass store (HPSS). # #----------------------------------------------------------------------- @@ -782,19 +782,23 @@ has not been specified for this external model: ;; "FV3GFS") + + if [ "${cdate_FV3LAM}" -lt "2019061200" ]; then + arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" + arcv_fns="" + elif [ "${cdate_FV3LAM}" -ge "2019061200" ] && \ + [ "${cdate_FV3LAM}" -lt "2020022600" ]; then + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." + elif [ "${cdate_FV3LAM}" -ge "2020022600" ]; then + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." + fi + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - if [ "${cdate_FV3LAM}" -le "2019061206" ]; then - arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" - arcv_fns="" - else - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." - fi - arcv_fmt="tar" + if [ "${anl_or_fcst}" = "ANL" ]; then arcv_fns="${arcv_fns}gfs_nemsioa" - arcvrel_dir="./gfs.${yyyymmdd}/${hh}" elif [ "${anl_or_fcst}" = "FCST" ]; then last_fhr_in_nemsioa="39" first_lbc_fhr="${lbc_spec_fhrs[0]}" @@ -806,18 +810,17 @@ has not been specified for this external model: else arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) fi - arcvrel_dir="./gfs.${yyyymmdd}/${hh}" fi elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}.gfs_pgrb2" - arcv_fmt="tar" - arcvrel_dir="./gfs.${yyyymmdd}/${hh}" - + arcv_fns="${arcv_fns}gfs_pgrb2" + fi + arcv_fmt="tar" + arcvrel_dir="./gfs.${yyyymmdd}/${hh}" + is_array arcv_fns if [ "$?" = "0" ]; then suffix=".${arcv_fmt}" From ae3d1a6184be7a6313dd109772173ff45ce87965 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Mon, 11 Jan 2021 10:24:10 -0700 Subject: [PATCH 006/203] Fix stochastic physics namelist block. Change tm${cyc} to tm00. (#387) * Fix stochastic physics block to get it working. Change tm${cyc} to tm00 in post script. * Add tmmark description. --- scripts/exregional_run_post.sh | 12 +++++++++++- ush/templates/input.nml.FV3 | 17 ++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 957c19c210..eaf79e12b3 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -185,7 +185,17 @@ cp_vrfy ${EXECDIR}/ncep_post . yyyymmdd=${cdate:0:8} hh=${cdate:8:2} cyc=$hh -tmmark="tm$hh" +# +#----------------------------------------------------------------------- +# +# The tmmark is a reference value used in real-time, DA-enabled NCEP models. +# It represents the delay between the onset of the DA cycle and the free +# forecast. With no DA in the SRW App at the moment, it is hard-wired to +# tm00 for now. +# +#----------------------------------------------------------------------- +# +tmmark="tm00" # #----------------------------------------------------------------------- # diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 5a02c71aff..d1f50229fc 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -197,17 +197,12 @@ / &nam_sfcperts - iseed_sfc = 0 - nsfcpert = 6 - pertalb = -999.0 - pertlai = -999.0 - pertshc = -999.0 - pertvegf = -999.0 - pertz0 = -999.0 - pertzt = -999.0 - sfc_lscale = 500000 - sfc_tau = 21600 - sppt_land = .false. + lndp_type = 0, + lndp_tau=21600, + lndp_lscale=500000, + iseed_lndp=2010, + lndp_var_list = 'smc', 'vgf', + lndp_prt_list = 0.2,0.1, / &nam_stochy From 4fd37fc852fa7b75342cee3ed0550093de471935 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Tue, 12 Jan 2021 11:53:59 -0700 Subject: [PATCH 007/203] develop: Namelist changes following GSL, EMC, and DTC namelist consolidation meeting. (#394) * Namelist changes following GSL, EMC, and DTC namelist consolidation meeting * Change ttendlim from 50 to -999 --- ush/templates/FV3.input.yml | 11 +++++------ ush/templates/input.nml.FV3 | 32 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 9f287aa78f..ac27188b71 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -16,8 +16,6 @@ FV3_GSD_v0: FV3_GSD_SAR: gfs_physics_nml: &gsd_sar_phys - cnvcld: false - cnvgwd: false effr_in: True imfdeepcnv: 0 imfshalcnv: 0 @@ -33,7 +31,6 @@ FV3_GSD_SAR: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - shal_cnv: false do_deep: False FV3_RRFS_v1alpha: @@ -85,8 +82,6 @@ FV3_GFS_2017_gfdlmp: bl_mynn_edmf: !!python/none bl_mynn_edmf_mom: !!python/none cdmbgwd: [3.5, 0.01] - cnvcld: False - cnvgwd: False cplflx: !!python/none do_deep: False do_mynnedmf: !!python/none @@ -111,7 +106,7 @@ FV3_GFS_2017_gfdlmp: oz_phys: True oz_phys_2015: False satmedmf: !!python/none - shal_cnv: False + shal_cnv: True ttendlim: !!python/none gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp c_cracw: 0.8 @@ -234,6 +229,8 @@ FV3_GFS_v15p2: bl_mynn_edmf: !!python/none bl_mynn_edmf_mom: !!python/none bl_mynn_tkeadvect: !!python/none + cnvcld: True + cnvgwd: True cplflx: !!python/none do_myjpbl: False do_myjsfc: False @@ -273,6 +270,7 @@ FV3_GFS_v15p2: lsoil: !!python/none lsoil_lsm: !!python/none ltaerosol: !!python/none + shal_cnv: True shinhong: False trans_trac: False ttendlim: !!python/none @@ -441,6 +439,7 @@ FV3_CPT_v0: random_clds: True ras: False rhcmax: 0.9999999 + shal_cnv: True shcnvcw: False shoc_cld: False shoc_parm: [7000.0, 1.0, 2.0, 0.7, -999.0] diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index d1f50229fc..64f9f4d05d 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -61,11 +61,11 @@ d2_bg = 0.0 d2_bg_k1 = 0.20 d2_bg_k2 = 0.04 - d4_bg = 0.12 + d4_bg = 0.15 d_con = 1.0 d_ext = 0.0 dddmp = 0.1 - delt_max = 0.002 + delt_max = 0.008 dnats = 0 do_sat_adj = .false. do_schmidt = .true. @@ -75,22 +75,22 @@ external_ic = .true. fill = .true. full_zs_filter = .false. - fv_debug = .true. + fv_debug = .false. fv_sg_adj = 300 gfs_phil = .false. - hord_dp = -6 + hord_dp = 6 hord_mt = 6 hord_tm = 6 - hord_tr = 8 + hord_tr = 10 hord_vt = 6 hydrostatic = .false. io_layout = 1,1 k_split = 4 ke_bg = 0.0 - kord_mt = 11 - kord_tm = -11 - kord_tr = 11 - kord_wz = 11 + kord_mt = 9 + kord_tm = -9 + kord_tr = 9 + kord_wz = 9 make_nh = .true. mountain = .false. n_split = 5 @@ -101,7 +101,6 @@ nggps_ic = .true. no_dycore = .false. nord = 2 - nord_zs_filter = 4 npz = 64 nrows_blend = 10 ntiles = 1 @@ -118,7 +117,7 @@ rf_cutoff = 20.e2 tau = 5.0 use_hydro_pressure = .false. - vtdm4 = 0.02 + vtdm4 = 0.075 warm_start = .false. z_tracer = .true. / @@ -133,8 +132,8 @@ bl_mynn_tkeadvect = .true. cal_pre = .false. cdmbgwd = 3.5,0.25 - cnvcld = .true. - cnvgwd = .true. + cnvcld = .false. + cnvgwd = .false. cplflx = .false. debug = .false. do_mynnedmf = .true. @@ -175,8 +174,9 @@ lwhtr = .true. n_var_lndp = 0 ncld = 5 + nsradar_reset = 3600 nst_anl = .true. - nstf_name = 2,1,1,0,5 + nstf_name = 2,1,0,0,0 oz_phys = .false. oz_phys_2015 = .true. pdfcld = .false. @@ -185,10 +185,10 @@ random_clds = .false. redrag = .true. satmedmf = .false. - shal_cnv = .true. + shal_cnv = .false. swhtr = .true. trans_trac = .true. - ttendlim = 50.0 + ttendlim = -999 use_ufo = .true. / From 14b914290ea09edbae95b060d3b1b3153f562f43 Mon Sep 17 00:00:00 2001 From: BenjaminBlake-NOAA <52074832+BenjaminBlake-NOAA@users.noreply.github.com> Date: Wed, 13 Jan 2021 08:07:15 -0500 Subject: [PATCH 008/203] Enhancements to Python plotting scripts (#381) * Updates to plot_allvars.py * Updated Instructions and Forced Forecast Interval to be INT - Updated instructions to include start fhr, end fhr, and interval. - Changed np.linspace to include "dtype='int'". This option requires NumPy v1.9.0 to work. - Added int() to the num calculation (~line 250) to also make it work with np.linspace. * Enhancements to plot_allvars_diff.py * Update plot_allvars.py * Update plot_allvars_diff.py Co-authored-by: David Wright --- ush/Python/plot_allvars.py | 708 ++++++++--------- ush/Python/plot_allvars_diff.py | 1270 +++++++++++++++---------------- 2 files changed, 995 insertions(+), 983 deletions(-) diff --git a/ush/Python/plot_allvars.py b/ush/Python/plot_allvars.py index d760ad372c..626b941abb 100755 --- a/ush/Python/plot_allvars.py +++ b/ush/Python/plot_allvars.py @@ -11,11 +11,13 @@ # Instructions: Make sure all the necessary modules can be imported. # Five command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format -# 2. Forecast hour in HHH format -# 3. EXPT_DIR: Experiment directory +# 2. Starting forecast hour +# 3. Ending forecast hour +# 4. Forecast hour increment +# 5. EXPT_DIR: Experiment directory # -Postprocessed data should be found in the directory: # EXPT_DIR/YYYYMMDDHH/postprd/ -# 4. CARTOPY_DIR: Base directory of cartopy shapefiles +# 6. CARTOPY_DIR: Base directory of cartopy shapefiles # -Shapefiles cannot be directly downloaded to NOAA # machines from the internet, so shapefiles need to # be downloaded if geopolitical boundaries are @@ -25,9 +27,11 @@ # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide # -# To create plots for forecast hour 24 from 5/7 00Z cycle: -# python plot_allvars.py 2020050700 24 /path/to/expt_dirs -# /experiment/name /path/to/base/cartopy/maps +# To create plots for forecast hours 20-24 from 5/7 00Z +# cycle with hourly output: +# python plot_allvars.py 2020050700 20 24 1 \ +# /path/to/expt_dirs/experiment/name \ +# /path/to/base/cartopy/maps # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -51,7 +55,6 @@ import time,os,sys,multiprocessing import multiprocessing.pool from scipy import ndimage -from netCDF4 import Dataset import pyproj import argparse import cartopy @@ -219,7 +222,9 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): # Define required positional arguments parser = argparse.ArgumentParser() parser.add_argument("Cycle date/time in YYYYMMDDHH format") -parser.add_argument("Forecast hour in HHH format") +parser.add_argument("Starting forecast hour") +parser.add_argument("Ending forecast hour") +parser.add_argument("Forecast hour increment") parser.add_argument("Path to experiment directory") parser.add_argument("Path to base directory of cartopy shapefiles") args = parser.parse_args() @@ -234,26 +239,38 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): cyc = str(hour).zfill(2) print(year, month, day, hour) -fhr = int(sys.argv[2]) -fhour = str(fhr).zfill(3) -print('fhour '+fhour) -itime = ymdh -vtime = ndate(itime,int(fhr)) - -EXPT_DIR = str(sys.argv[3]) -CARTOPY_DIR = str(sys.argv[4]) +# Define the range of forecast hours to create plots for +start_fhr = int(sys.argv[2]) +end_fhr = int(sys.argv[3]) +increment_fhr = int(sys.argv[4]) +if (start_fhr == end_fhr) or (increment_fhr == 0): + fhours = [start_fhr] +else: + num = int(((end_fhr - start_fhr) / increment_fhr) + 1) + fhours = np.linspace(start_fhr,end_fhr,num,dtype='int') +print(fhours) + +EXPT_DIR = str(sys.argv[5]) +CARTOPY_DIR = str(sys.argv[6]) + +# Loop over forecast hours +for fhr in fhours: + fhour = str(fhr).zfill(3) + print('Working on forecast hour '+fhour) + itime = ymdh + vtime = ndate(itime,int(fhr)) # Define the location of the input file -data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') # Get the lats and lons -grids = [data1] -lats = [] -lons = [] -lats_shift = [] -lons_shift = [] + grids = [data1] + lats = [] + lons = [] + lats_shift = [] + lons_shift = [] -for data in grids: + for data in grids: # Unshifted grid for contours and wind barbs lat, lon = data[1].latlons() lats.append(lat) @@ -282,503 +299,502 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): lons_shift.append(lon) # Unshifted lat/lon arrays grabbed directly using latlons() method -lat = lats[0] -lon = lons[0] + lat = lats[0] + lon = lons[0] # Shifted lat/lon arrays for pcolormesh -lat_shift = lats_shift[0] -lon_shift = lons_shift[0] + lat_shift = lats_shift[0] + lon_shift = lons_shift[0] -Lat0 = data1[1]['LaDInDegrees'] -Lon0 = data1[1]['LoVInDegrees'] -print(Lat0) -print(Lon0) + Lat0 = data1[1]['LaDInDegrees'] + Lon0 = data1[1]['LoVInDegrees'] + print(Lat0) + print(Lon0) # Specify plotting domains # User can add domains here, just need to specify lat/lon information below # (if dom == 'conus' block) -domains=['conus'] # Other option is 'regional' + domains=['conus'] # Other option is 'regional' ################################################### # Read in all variables and calculate differences # ################################################### -t1a = time.perf_counter() + t1a = time.perf_counter() # Sea level pressure -slp = data1.select(name='Pressure reduced to MSL')[0].values * 0.01 -slpsmooth = ndimage.filters.gaussian_filter(slp, 13.78) + slp = data1.select(name='Pressure reduced to MSL')[0].values * 0.01 + slpsmooth = ndimage.filters.gaussian_filter(slp, 13.78) # 2-m temperature -tmp2m = data1.select(name='2 metre temperature')[0].values -tmp2m = (tmp2m - 273.15)*1.8 + 32.0 + tmp2m = data1.select(name='2 metre temperature')[0].values + tmp2m = (tmp2m - 273.15)*1.8 + 32.0 # 2-m dew point temperature -dew2m = data1.select(name='2 metre dewpoint temperature')[0].values -dew2m = (dew2m - 273.15)*1.8 + 32.0 + dew2m = data1.select(name='2 metre dewpoint temperature')[0].values + dew2m = (dew2m - 273.15)*1.8 + 32.0 # 10-m wind speed -uwind = data1.select(name='10 metre U wind component')[0].values * 1.94384 -vwind = data1.select(name='10 metre V wind component')[0].values * 1.94384 + uwind = data1.select(name='10 metre U wind component')[0].values * 1.94384 + vwind = data1.select(name='10 metre V wind component')[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -uwind, vwind = rotate_wind(Lat0,Lon0,lon,uwind,vwind,'lcc',inverse=False) -wspd10m = np.sqrt(uwind**2 + vwind**2) + uwind, vwind = rotate_wind(Lat0,Lon0,lon,uwind,vwind,'lcc',inverse=False) + wspd10m = np.sqrt(uwind**2 + vwind**2) # Surface-based CAPE -cape = data1.select(name='Convective available potential energy',typeOfLevel='surface')[0].values + cape = data1.select(name='Convective available potential energy',typeOfLevel='surface')[0].values # Surface-based CIN -cin = data1.select(name='Convective inhibition',typeOfLevel='surface')[0].values + cin = data1.select(name='Convective inhibition',typeOfLevel='surface')[0].values # 500 mb height, wind, vorticity -z500 = data1.select(name='Geopotential Height',level=500)[0].values * 0.1 -z500 = ndimage.filters.gaussian_filter(z500, 6.89) -vort500 = data1.select(name='Absolute vorticity',level=500)[0].values * 100000 -vort500 = ndimage.filters.gaussian_filter(vort500,1.7225) -vort500[vort500 > 1000] = 0 # Mask out undefined values on domain edge -u500 = data1.select(name='U component of wind',level=500)[0].values * 1.94384 -v500 = data1.select(name='V component of wind',level=500)[0].values * 1.94384 + z500 = data1.select(name='Geopotential Height',level=500)[0].values * 0.1 + z500 = ndimage.filters.gaussian_filter(z500, 6.89) + vort500 = data1.select(name='Absolute vorticity',level=500)[0].values * 100000 + vort500 = ndimage.filters.gaussian_filter(vort500,1.7225) + vort500[vort500 > 1000] = 0 # Mask out undefined values on domain edge + u500 = data1.select(name='U component of wind',level=500)[0].values * 1.94384 + v500 = data1.select(name='V component of wind',level=500)[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -u500, v500 = rotate_wind(Lat0,Lon0,lon,u500,v500,'lcc',inverse=False) + u500, v500 = rotate_wind(Lat0,Lon0,lon,u500,v500,'lcc',inverse=False) # 250 mb winds -u250 = data1.select(name='U component of wind',level=250)[0].values * 1.94384 -v250 = data1.select(name='V component of wind',level=250)[0].values * 1.94384 + u250 = data1.select(name='U component of wind',level=250)[0].values * 1.94384 + v250 = data1.select(name='V component of wind',level=250)[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -u250, v250 = rotate_wind(Lat0,Lon0,lon,u250,v250,'lcc',inverse=False) -wspd250 = np.sqrt(u250**2 + v250**2) + u250, v250 = rotate_wind(Lat0,Lon0,lon,u250,v250,'lcc',inverse=False) + wspd250 = np.sqrt(u250**2 + v250**2) # Total precipitation -qpf = data1.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 + qpf = data1.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 # Composite reflectivity -refc = data1.select(name='Maximum/Composite radar reflectivity')[0].values + refc = data1.select(name='Maximum/Composite radar reflectivity')[0].values -if (fhr > 0): + if (fhr > 0): # Max/Min Hourly 2-5 km Updraft Helicity - maxuh25 = data1.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values - minuh25 = data1.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values - maxuh25[maxuh25 < 10] = 0 - minuh25[minuh25 > -10] = 0 - uh25 = maxuh25 + minuh25 + maxuh25 = data1.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values + minuh25 = data1.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values + maxuh25[maxuh25 < 10] = 0 + minuh25[minuh25 > -10] = 0 + uh25 = maxuh25 + minuh25 -t2a = time.perf_counter() -t3a = round(t2a-t1a, 3) -print(("%.3f seconds to read all messages") % t3a) + t2a = time.perf_counter() + t3a = round(t2a-t1a, 3) + print(("%.3f seconds to read all messages") % t3a) ######################################## # START PLOTTING FOR EACH DOMAIN # ######################################## -def main(): + def main(): # Number of processes must coincide with the number of domains to plot - pool = multiprocessing.Pool(len(domains)) - pool.map(plot_all,domains) + pool = multiprocessing.Pool(len(domains)) + pool.map(plot_all,domains) -def plot_all(dom): + def plot_all(dom): - t1dom = time.perf_counter() - print(('Working on '+dom)) + t1dom = time.perf_counter() # Map corners for each domain - if dom == 'conus': - llcrnrlon = -120.5 - llcrnrlat = 21.0 - urcrnrlon = -64.5 - urcrnrlat = 49.0 - lat_0 = 35.4 - lon_0 = -97.6 - extent=[llcrnrlon-3,urcrnrlon-6,llcrnrlat-1,urcrnrlat+2] - elif dom == 'regional': - llcrnrlon = np.min(lon) - llcrnrlat = np.min(lat) - urcrnrlon = np.max(lon) - urcrnrlat = np.max(lat) - lat_0 = Lat0 - lon_0 = Lon0 - extent=[llcrnrlon,urcrnrlon,llcrnrlat-1,urcrnrlat] + if dom == 'conus': + llcrnrlon = -120.5 + llcrnrlat = 21.0 + urcrnrlon = -64.5 + urcrnrlat = 49.0 + lat_0 = 35.4 + lon_0 = -97.6 + extent=[llcrnrlon-3,urcrnrlon-6,llcrnrlat-1,urcrnrlat+2] + elif dom == 'regional': + llcrnrlon = np.min(lon) + llcrnrlat = np.min(lat) + urcrnrlon = np.max(lon) + urcrnrlat = np.max(lat) + lat_0 = Lat0 + lon_0 = Lon0 + extent=[llcrnrlon,urcrnrlon,llcrnrlat-1,urcrnrlat] # create figure and axes instances - fig = plt.figure(figsize=(10,10)) - ax1 = fig.add_axes([0.1,0.1,0.8,0.8]) + fig = plt.figure(figsize=(10,10)) + ax1 = fig.add_axes([0.1,0.1,0.8,0.8]) # Define where Cartopy Maps are located - cartopy.config['data_dir'] = CARTOPY_DIR + cartopy.config['data_dir'] = CARTOPY_DIR - back_res='50m' - back_img='on' + back_res='50m' + back_img='on' # set up the map background with cartopy - myproj=ccrs.LambertConformal(central_longitude=lon_0, central_latitude=lat_0, false_easting=0.0, - false_northing=0.0, secant_latitudes=None, standard_parallels=None, - globe=None) - ax = plt.axes(projection=myproj) - ax.set_extent(extent) + myproj=ccrs.LambertConformal(central_longitude=lon_0, central_latitude=lat_0, false_easting=0.0, + false_northing=0.0, secant_latitudes=None, standard_parallels=None, + globe=None) + ax = plt.axes(projection=myproj) + ax.set_extent(extent) - fline_wd = 0.5 # line width - falpha = 0.3 # transparency + fline_wd = 0.5 # line width + falpha = 0.3 # transparency # natural_earth # land=cfeature.NaturalEarthFeature('physical','land',back_res, # edgecolor='face',facecolor=cfeature.COLORS['land'], # alpha=falpha) - lakes=cfeature.NaturalEarthFeature('physical','lakes',back_res, - edgecolor='blue',facecolor='none', - linewidth=fline_wd,alpha=falpha) - coastline=cfeature.NaturalEarthFeature('physical','coastline', - back_res,edgecolor='blue',facecolor='none', - linewidth=fline_wd,alpha=falpha) - states=cfeature.NaturalEarthFeature('cultural','admin_1_states_provinces', - back_res,edgecolor='black',facecolor='none', - linewidth=fline_wd,linestyle=':',alpha=falpha) - borders=cfeature.NaturalEarthFeature('cultural','admin_0_countries', - back_res,edgecolor='red',facecolor='none', - linewidth=fline_wd,alpha=falpha) + lakes=cfeature.NaturalEarthFeature('physical','lakes',back_res, + edgecolor='blue',facecolor='none', + linewidth=fline_wd,alpha=falpha) + coastline=cfeature.NaturalEarthFeature('physical','coastline', + back_res,edgecolor='blue',facecolor='none', + linewidth=fline_wd,alpha=falpha) + states=cfeature.NaturalEarthFeature('cultural','admin_1_states_provinces', + back_res,edgecolor='black',facecolor='none', + linewidth=fline_wd,linestyle=':',alpha=falpha) + borders=cfeature.NaturalEarthFeature('cultural','admin_0_countries', + back_res,edgecolor='red',facecolor='none', + linewidth=fline_wd,alpha=falpha) # All lat lons are earth relative, so setup the associated projection correct for that data - transform = ccrs.PlateCarree() + transform = ccrs.PlateCarree() # high-resolution background images - if back_img=='on': - img = plt.imread(CARTOPY_DIR+'/raster_files/NE1_50M_SR_W.tif') - ax.imshow(img, origin='upper', transform=transform) + if back_img=='on': + img = plt.imread(CARTOPY_DIR+'/raster_files/NE1_50M_SR_W.tif') + ax.imshow(img, origin='upper', transform=transform) # ax.add_feature(land) - ax.add_feature(lakes) - ax.add_feature(states) - ax.add_feature(borders) - ax.add_feature(coastline) + ax.add_feature(lakes) + ax.add_feature(states) + ax.add_feature(borders) + ax.add_feature(coastline) # Map/figure has been set up here, save axes instances for use again later - keep_ax_lst = ax.get_children()[:] + keep_ax_lst = ax.get_children()[:] ################################ # Plot SLP ################################ - t1 = time.perf_counter() - print(('Working on slp for '+dom)) + t1 = time.perf_counter() + print(('Working on slp for '+dom)) - units = 'mb' - clevs = [976,980,984,988,992,996,1000,1004,1008,1012,1016,1020,1024,1028,1032,1036,1040,1044,1048,1052] - clevsdif = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] - cm = plt.cm.Spectral_r - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + units = 'mb' + clevs = [976,980,984,988,992,996,1000,1004,1008,1012,1016,1020,1024,1028,1032,1036,1040,1044,1048,1052] + clevsdif = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] + cm = plt.cm.Spectral_r + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs1_a = plt.pcolormesh(lon_shift,lat_shift,slp,transform=transform,cmap=cm,norm=norm) - cbar1 = plt.colorbar(cs1_a,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - cs1_b = plt.contour(lon_shift,lat_shift,slpsmooth,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) - plt.clabel(cs1_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=8) - ax.text(.5,1.03,'FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + cs1_a = plt.pcolormesh(lon_shift,lat_shift,slp,transform=transform,cmap=cm,norm=norm) + cbar1 = plt.colorbar(cs1_a,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + cs1_b = plt.contour(lon_shift,lat_shift,slpsmooth,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) + plt.clabel(cs1_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=8) + ax.text(.5,1.03,'FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('slp_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot slp for: '+dom) % t3) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/slp_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot slp for: '+dom) % t3) ################################# # Plot 2-m T ################################# - t1 = time.perf_counter() - print(('Working on t2m for '+dom)) + t1 = time.perf_counter() + print(('Working on t2m for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) - units = '\xb0''F' - clevs = np.linspace(-16,134,51) - cm = plt.cm.Spectral_r #cmap_t2m() - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + units = '\xb0''F' + clevs = np.linspace(-16,134,51) + cm = plt.cm.Spectral_r #cmap_t2m() + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs_1 = plt.pcolormesh(lon_shift,lat_shift,tmp2m,transform=transform,cmap=cm,norm=norm) - cs_1.cmap.set_under('white') - cs_1.cmap.set_over('white') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - ax.text(.5,1.03,'FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + cs_1 = plt.pcolormesh(lon_shift,lat_shift,tmp2m,transform=transform,cmap=cm,norm=norm) + cs_1.cmap.set_under('white') + cs_1.cmap.set_over('white') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + ax.text(.5,1.03,'FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('2mt_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 2mt for: '+dom) % t3) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/2mt_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 2mt for: '+dom) % t3) ################################# # Plot 2-m Dew Point ################################# - t1 = time.perf_counter() - print(('Working on 2mdew for '+dom)) + t1 = time.perf_counter() + print(('Working on 2mdew for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) - units = '\xb0''F' - clevs = np.linspace(-5,80,35) - cm = cmap_q2m() - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + units = '\xb0''F' + clevs = np.linspace(-5,80,35) + cm = cmap_q2m() + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs_1 = plt.pcolormesh(lon_shift,lat_shift,dew2m,transform=transform,cmap=cm,norm=norm) - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - ax.text(.5,1.03,'FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + cs_1 = plt.pcolormesh(lon_shift,lat_shift,dew2m,transform=transform,cmap=cm,norm=norm) + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + ax.text(.5,1.03,'FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('2mdew_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 2mdew for: '+dom) % t3) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/2mdew_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 2mdew for: '+dom) % t3) ################################# # Plot 10-m WSPD ################################# - t1 = time.perf_counter() - print(('Working on 10mwspd for '+dom)) + t1 = time.perf_counter() + print(('Working on 10mwspd for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) - units = 'kts' + units = 'kts' # Places a wind barb every ~180 km, optimized for CONUS domain - skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') - barblength = 4 - - clevs = [5,10,15,20,25,30,35,40,45,50,55,60] - colorlist = ['turquoise','dodgerblue','blue','#FFF68F','#E3CF57','peru','brown','crimson','red','fuchsia','DarkViolet'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - - cs_1 = plt.pcolormesh(lon_shift,lat_shift,wspd10m,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],uwind[::skip,::skip],vwind[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax.text(.5,1.03,'FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + skip = round(177.28*(dx/1000.)**-.97) + print('skipping every '+str(skip)+' grid points to plot') + barblength = 4 + + clevs = [5,10,15,20,25,30,35,40,45,50,55,60] + colorlist = ['turquoise','dodgerblue','blue','#FFF68F','#E3CF57','peru','brown','crimson','red','fuchsia','DarkViolet'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + + cs_1 = plt.pcolormesh(lon_shift,lat_shift,wspd10m,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('black') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],uwind[::skip,::skip],vwind[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax.text(.5,1.03,'FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('10mwind_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 10mwspd for: '+dom) % t3) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/10mwind_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 10mwspd for: '+dom) % t3) ################################# # Plot Surface-Based CAPE/CIN ################################# - t1 = time.perf_counter() - print(('Working on surface-based CAPE/CIN for '+dom)) + t1 = time.perf_counter() + print(('Working on surface-based CAPE/CIN for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) - - units = 'J/kg' - clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] - clevs2 = [-2000,-500,-250,-100,-25] - colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - - cs_1 = plt.pcolormesh(lon_shift,lat_shift,cape,transform=transform,cmap=cm,vmin=100,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('darkviolet') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - cs_1b = plt.contourf(lon_shift,lat_shift,cin,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) - ax.text(.5,1.05,'FV3-LAM Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('sfcape_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot surface-based CAPE/CIN for: '+dom) % t3) + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) + units = 'J/kg' + clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] + clevs2 = [-2000,-500,-250,-100,-25] + colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) -################################# - # Plot 500 mb HGT/WIND/VORT -################################# - t1 = time.perf_counter() - print(('Working on 500 mb Hgt/Wind/Vort for '+dom)) + cs_1 = plt.pcolormesh(lon_shift,lat_shift,cape,transform=transform,cmap=cm,vmin=100,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('darkviolet') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + cs_1b = plt.contourf(lon_shift,lat_shift,cin,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) + ax.text(.5,1.05,'FV3-LAM Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) - - units = 'x10${^5}$ s${^{-1}}$' - skip = round(177.28*(dx/1000.)**-.97) - barblength = 4 - - vortlevs = [16,20,24,28,32,36,40] - colorlist = ['yellow','gold','goldenrod','orange','orangered','red'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(vortlevs, cm.N) - - cs1_a = plt.pcolormesh(lon_shift,lat_shift,vort500,transform=transform,cmap=cm,norm=norm) - cs1_a.cmap.set_under('white') - cs1_a.cmap.set_over('darkred') - cbar1 = plt.colorbar(cs1_a,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u500[::skip,::skip],v500[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) - cs1_b = plt.contour(lon_shift,lat_shift,z500,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) - plt.clabel(cs1_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) - ax.text(.5,1.03,'FV3-LAM 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('500_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 500 mb Hgt/Wind/Vort for: '+dom) % t3) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/sfcape_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot surface-based CAPE/CIN for: '+dom) % t3) ################################# - # Plot 250 mb WIND + # Plot 500 mb HGT/WIND/VORT ################################# - t1 = time.perf_counter() - print(('Working on 250 mb WIND for '+dom)) + t1 = time.perf_counter() + print(('Working on 500 mb Hgt/Wind/Vort for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) - units = 'kts' - skip = round(177.28*(dx/1000.)**-.97) + units = 'x10${^5}$ s${^{-1}}$' + skip = round(177.28*(dx/1000.)**-.97) + barblength = 4 - barblength = 4 + vortlevs = [16,20,24,28,32,36,40] + colorlist = ['yellow','gold','goldenrod','orange','orangered','red'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(vortlevs, cm.N) - clevs = [50,60,70,80,90,100,110,120,130,140,150] - colorlist = ['turquoise','deepskyblue','dodgerblue','#1874CD','blue','beige','khaki','peru','brown','crimson'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + cs1_a = plt.pcolormesh(lon_shift,lat_shift,vort500,transform=transform,cmap=cm,norm=norm) + cs1_a.cmap.set_under('white') + cs1_a.cmap.set_over('darkred') + cbar1 = plt.colorbar(cs1_a,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u500[::skip,::skip],v500[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) + cs1_b = plt.contour(lon_shift,lat_shift,z500,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) + plt.clabel(cs1_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) + ax.text(.5,1.03,'FV3-LAM 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - cs_1 = plt.pcolormesh(lon_shift,lat_shift,wspd250,transform=transform,cmap=cm,vmin=50,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('red') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u250[::skip,::skip],v250[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax.text(.5,1.03,'FV3-LAM 250 mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/500_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 500 mb Hgt/Wind/Vort for: '+dom) % t3) - compress_and_save('250wind_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 250 mb WIND for: '+dom) % t3) ################################# - # Plot Total QPF + # Plot 250 mb WIND ################################# - if (fhr > 0): # Do not make total QPF plot for forecast hour 0 t1 = time.perf_counter() - print(('Working on total qpf for '+dom)) + print(('Working on 250 mb WIND for '+dom)) - # Clear off old plottables but keep all the map info + # Clear off old plottables but keep all the map info cbar1.remove() clear_plotables(ax,keep_ax_lst,fig) - units = 'in' - clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] - clevsdif = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] - colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + units = 'kts' + skip = round(177.28*(dx/1000.)**-.97) + + barblength = 4 + + clevs = [50,60,70,80,90,100,110,120,130,140,150] + colorlist = ['turquoise','deepskyblue','dodgerblue','#1874CD','blue','beige','khaki','peru','brown','crimson'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs_1 = plt.pcolormesh(lon_shift,lat_shift,qpf,transform=transform,cmap=cm,vmin=0.01,norm=norm) + cs_1 = plt.pcolormesh(lon_shift,lat_shift,wspd250,transform=transform,cmap=cm,vmin=50,norm=norm) cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('pink') + cs_1.cmap.set_over('red') cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar1.set_label(units,fontsize=8) - cbar1.ax.set_xticklabels(clevs) cbar1.ax.tick_params(labelsize=8) - ax.text(.5,1.03,'FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u250[::skip,::skip],v250[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax.text(.5,1.03,'FV3-LAM 250 mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('qpf_'+dom+'_f'+fhour+'.png') + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/250wind_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) - print(('%.3f seconds to plot total qpf for: '+dom) % t3) + print(('%.3f seconds to plot 250 mb WIND for: '+dom) % t3) ################################# - # Plot composite reflectivity + # Plot Total QPF ################################# - t1 = time.perf_counter() - print(('Working on composite reflectivity for '+dom)) - - # Clear off old plottables but keep all the map info - cbar1.remove() - clear_plotables(ax,keep_ax_lst,fig) - - units = 'dBZ' - clevs = np.linspace(5,70,14) - clevsdif = [20,1000] - colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - - cs_1 = plt.pcolormesh(lon_shift,lat_shift,refc,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=8) - cbar1.ax.tick_params(labelsize=8) - ax.text(.5,1.03,'FV3-LAM Composite Reflectivity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('refc_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot composite reflectivity for: '+dom) % t3) + if (fhr > 0): # Do not make total QPF plot for forecast hour 0 + t1 = time.perf_counter() + print(('Working on total qpf for '+dom)) + # Clear off old plottables but keep all the map info + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) + + units = 'in' + clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] + clevsdif = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] + colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + + cs_1 = plt.pcolormesh(lon_shift,lat_shift,qpf,transform=transform,cmap=cm,vmin=0.01,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('pink') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=8) + cbar1.ax.set_xticklabels(clevs) + cbar1.ax.tick_params(labelsize=8) + ax.text(.5,1.03,'FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/qpf_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot total qpf for: '+dom) % t3) ################################# - # Plot Max/Min Hourly 2-5 km UH + # Plot composite reflectivity ################################# - if (fhr > 0): # Do not make max/min hourly 2-5 km UH plot for forecast hour 0 t1 = time.perf_counter() - print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) + print(('Working on composite reflectivity for '+dom)) - # Clear off old plottables but keep all the map info + # Clear off old plottables but keep all the map info cbar1.remove() clear_plotables(ax,keep_ax_lst,fig) - units = 'm${^2}$ s$^{-2}$' - clevs = [-150,-100,-75,-50,-25,-10,0,10,25,50,75,100,150,200,250,300] -# alternative colormap for just max UH if you don't want to plot the min UH too -# colorlist = ['white','skyblue','mediumblue','green','orchid','firebrick','#EEC900','DarkViolet'] - colorlist = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','#E5E5E5','#E5E5E5','#EEEE00','#EEC900','darkorange','orangered','red','firebrick','mediumvioletred','darkviolet'] + units = 'dBZ' + clevs = np.linspace(5,70,14) + clevsdif = [20,1000] + colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - - cs_1 = plt.pcolormesh(lon_shift,lat_shift,uh25,transform=transform,cmap=cm,norm=norm) - cs_1.cmap.set_under('darkblue') + + cs_1 = plt.pcolormesh(lon_shift,lat_shift,refc,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar1.set_label(units,fontsize=8) cbar1.ax.tick_params(labelsize=8) - ax.text(.5,1.03,'FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax.text(.5,1.03,'FV3-LAM Composite Reflectivity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('uh25_'+dom+'_f'+fhour+'.png') + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/refc_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) - print(('%.3f seconds to plot Max/Min Hourly 2-5 km UH for: '+dom) % t3) + print(('%.3f seconds to plot composite reflectivity for: '+dom) % t3) + + +################################# + # Plot Max/Min Hourly 2-5 km UH +################################# + if (fhr > 0): # Do not make max/min hourly 2-5 km UH plot for forecast hour 0 + t1 = time.perf_counter() + print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) + + # Clear off old plottables but keep all the map info + cbar1.remove() + clear_plotables(ax,keep_ax_lst,fig) + + units = 'm${^2}$ s$^{-2}$' + clevs = [-150,-100,-75,-50,-25,-10,0,10,25,50,75,100,150,200,250,300] +# alternative colormap for just max UH if you don't want to plot the min UH too +# colorlist = ['white','skyblue','mediumblue','green','orchid','firebrick','#EEC900','DarkViolet'] + colorlist = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','#E5E5E5','#E5E5E5','#EEEE00','#EEC900','darkorange','orangered','red','firebrick','mediumvioletred','darkviolet'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + + cs_1 = plt.pcolormesh(lon_shift,lat_shift,uh25,transform=transform,cmap=cm,norm=norm) + cs_1.cmap.set_under('darkblue') + cs_1.cmap.set_over('black') + cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=8) + cbar1.ax.tick_params(labelsize=8) + ax.text(.5,1.03,'FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/uh25_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot Max/Min Hourly 2-5 km UH for: '+dom) % t3) ###################################################### - t3dom = round(t2-t1dom, 3) - print(("%.3f seconds to plot all variables for: "+dom) % t3dom) - plt.clf() + t3dom = round(t2-t1dom, 3) + print(("%.3f seconds to plot all variables for forecast hour "+fhour) % t3dom) + plt.clf() ###################################################### -main() + main() diff --git a/ush/Python/plot_allvars_diff.py b/ush/Python/plot_allvars_diff.py index 24be6df6ad..a44a147e9c 100755 --- a/ush/Python/plot_allvars_diff.py +++ b/ush/Python/plot_allvars_diff.py @@ -10,13 +10,13 @@ # # Instructions: Make sure all the necessary modules can be imported. # Seven command line arguments are needed: -# 1. Cycle date/time in YYYYMMDDHH format for 1st file -# 2. Forecast hour in HHH format for 1st file -# 3. EXPT_DIR_1: Experiment 1 directory +# 1. Cycle date/time in YYYYMMDDHH format +# 2. Starting forecast hour +# 3. Ending forecast hour +# 4. Forecast hour increment +# 5. EXPT_DIR_1: Experiment 1 directory # -Postprocessed data should be found in the directory: # EXPT_DIR_1/YYYYMMDDHH/postprd/ -# 4. Cycle date/time in YYYYMMDDHH format for 2nd file -# 5. Forecast hour in HHH format for 2nd file # 6. EXPT_DIR_2: Experiment 2 directory # -Postprocessed data should be found in the directory: # EXPT_DIR_2/YYYYMMDDHH/postprd/ @@ -30,10 +30,11 @@ # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide # -# To create plots for forecast hour 24 from 5/7 00Z cycle: -# python plot_allvars.py 2020050700 024 /path/to/expt_dir_1 -# 2020050700 024 /path/to/expt_dir_2 -# /path/to/base/cartopy/maps +# To create plots for forecast hours 20-24 from 5/7 00Z +# cycle with hourly output: +# python plot_allvars_diff.py 2020050700 20 24 1 \ +# /path/to/expt_dir_1 /path/to/expt_dir_2 \ +# /path/to/base/cartopy/maps # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -57,7 +58,6 @@ import time,os,sys,multiprocessing import multiprocessing.pool from scipy import ndimage -from netCDF4 import Dataset import pyproj import argparse import cartopy @@ -224,11 +224,11 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): # Define required positional arguments parser = argparse.ArgumentParser() -parser.add_argument("Cycle date/time in YYYYMMDDHH format for 1st file") -parser.add_argument("Forecast hour in HHH format for 1st file") +parser.add_argument("Cycle date/time in YYYYMMDDHH format") +parser.add_argument("Starting forecast hour") +parser.add_argument("Ending forecast hour") +parser.add_argument("Forecast hour increment") parser.add_argument("Path to experiment 1 directory") -parser.add_argument("Cycle date/time in YYYYMMDDHH format for 2nd file") -parser.add_argument("Forecast hour in HHH format for 2nd file") parser.add_argument("Path to experiment 2 directory") parser.add_argument("Path to base directory of cartopy shapefiles") args = parser.parse_args() @@ -243,44 +243,41 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): cyc = str(hour).zfill(2) print(year, month, day, hour) -fhr = int(sys.argv[2]) -fhour = str(fhr).zfill(3) -print('fhour '+fhour) -itime = ymdh -vtime = ndate(itime,int(fhr)) - -EXPT_DIR_1 = str(sys.argv[3]) - -ymdh2 = str(sys.argv[4]) -ymd2 = ymdh2[0:8] -year2 = int(ymdh2[0:4]) -month2 = int(ymdh2[4:6]) -day2 = int(ymdh2[6:8]) -hour2 = int(ymdh2[8:10]) -cyc2 = str(hour2).zfill(2) -print(year2, month2, day2, hour2) - -fhr2 = int(sys.argv[5]) -fhour2 = str(fhr2).zfill(3) -print('fhour '+fhour2) -itime2 = ymdh2 -vtime2 = ndate(itime2,int(fhr2)) - +# Define the range of forecast hours to create plots for +start_fhr = int(sys.argv[2]) +end_fhr = int(sys.argv[3]) +increment_fhr = int(sys.argv[4]) +if (start_fhr == end_fhr) or (increment_fhr == 0): + fhours = [start_fhr] +else: + num = int(((end_fhr - start_fhr) / increment_fhr) + 1) + fhours = np.linspace(start_fhr,end_fhr,num,dtype='int') +print(fhours) + +EXPT_DIR_1 = str(sys.argv[5]) EXPT_DIR_2 = str(sys.argv[6]) CARTOPY_DIR = str(sys.argv[7]) +# Loop over forecast hours +for fhr in fhours: + fhour = str(fhr).zfill(3) + print('Working on forecast hour '+fhour) + itime = ymdh + vtime = ndate(itime,int(fhr)) + + # Define the location of the input files -data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') -data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh2+'/postprd/rrfs.t'+cyc2+'z.bgdawpf'+fhour2+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') + data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') # Get the lats and lons -grids = [data1, data2] -lats = [] -lons = [] -lats_shift = [] -lons_shift = [] + grids = [data1, data2] + lats = [] + lons = [] + lats_shift = [] + lons_shift = [] -for data in grids: + for data in grids: # Unshifted grid for contours and wind barbs lat, lon = data[1].latlons() lats.append(lat) @@ -309,629 +306,507 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): lons_shift.append(lon) # Unshifted lat/lon arrays grabbed directly using latlons() method -lat = lats[0] -lon = lons[0] -lat2 = lats[1] -lon2 = lons[1] + lat = lats[0] + lon = lons[0] + lat2 = lats[1] + lon2 = lons[1] # Shifted lat/lon arrays for pcolormesh -lat_shift = lats_shift[0] -lon_shift = lons_shift[0] -lat2_shift = lats_shift[1] -lon2_shift = lons_shift[1] + lat_shift = lats_shift[0] + lon_shift = lons_shift[0] + lat2_shift = lats_shift[1] + lon2_shift = lons_shift[1] -Lat0 = data1[1]['LaDInDegrees'] -Lon0 = data1[1]['LoVInDegrees'] -print(Lat0) -print(Lon0) + Lat0 = data1[1]['LaDInDegrees'] + Lon0 = data1[1]['LoVInDegrees'] + print(Lat0) + print(Lon0) # Specify plotting domains # User can add domains here, just need to specify lat/lon information below # (if dom == 'conus' block) -domains=['conus'] # Other option is 'regional' + domains=['conus'] # Other option is 'regional' ################################################### # Read in all variables and calculate differences # ################################################### -t1a = time.perf_counter() + t1a = time.perf_counter() # Sea level pressure -slp_1 = data1.select(name='Pressure reduced to MSL')[0].values * 0.01 -slpsmooth_1 = ndimage.filters.gaussian_filter(slp_1, 13.78) -slp_2 = data2.select(name='Pressure reduced to MSL')[0].values * 0.01 -slpsmooth_2 = ndimage.filters.gaussian_filter(slp_2, 13.78) -slp_diff = slp_2 - slp_1 + slp_1 = data1.select(name='Pressure reduced to MSL')[0].values * 0.01 + slpsmooth_1 = ndimage.filters.gaussian_filter(slp_1, 13.78) + slp_2 = data2.select(name='Pressure reduced to MSL')[0].values * 0.01 + slpsmooth_2 = ndimage.filters.gaussian_filter(slp_2, 13.78) + slp_diff = slp_2 - slp_1 # 2-m temperature -tmp2m_1 = data1.select(name='2 metre temperature')[0].values -tmp2m_1 = (tmp2m_1 - 273.15)*1.8 + 32.0 -tmp2m_2 = data2.select(name='2 metre temperature')[0].values -tmp2m_2 = (tmp2m_2 - 273.15)*1.8 + 32.0 -tmp2m_diff = tmp2m_2 - tmp2m_1 + tmp2m_1 = data1.select(name='2 metre temperature')[0].values + tmp2m_1 = (tmp2m_1 - 273.15)*1.8 + 32.0 + tmp2m_2 = data2.select(name='2 metre temperature')[0].values + tmp2m_2 = (tmp2m_2 - 273.15)*1.8 + 32.0 + tmp2m_diff = tmp2m_2 - tmp2m_1 # 2-m dew point temperature -dew2m_1 = data1.select(name='2 metre dewpoint temperature')[0].values -dew2m_1 = (dew2m_1 - 273.15)*1.8 + 32.0 -dew2m_2 = data2.select(name='2 metre dewpoint temperature')[0].values -dew2m_2 = (dew2m_2 - 273.15)*1.8 + 32.0 -dew2m_diff = dew2m_2 - dew2m_1 + dew2m_1 = data1.select(name='2 metre dewpoint temperature')[0].values + dew2m_1 = (dew2m_1 - 273.15)*1.8 + 32.0 + dew2m_2 = data2.select(name='2 metre dewpoint temperature')[0].values + dew2m_2 = (dew2m_2 - 273.15)*1.8 + 32.0 + dew2m_diff = dew2m_2 - dew2m_1 # 10-m wind speed -uwind_1 = data1.select(name='10 metre U wind component')[0].values * 1.94384 -vwind_1 = data1.select(name='10 metre V wind component')[0].values * 1.94384 -uwind_2 = data2.select(name='10 metre U wind component')[0].values * 1.94384 -vwind_2 = data2.select(name='10 metre V wind component')[0].values * 1.94384 + uwind_1 = data1.select(name='10 metre U wind component')[0].values * 1.94384 + vwind_1 = data1.select(name='10 metre V wind component')[0].values * 1.94384 + uwind_2 = data2.select(name='10 metre U wind component')[0].values * 1.94384 + vwind_2 = data2.select(name='10 metre V wind component')[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -uwind_1, vwind_1 = rotate_wind(Lat0,Lon0,lon,uwind_1,vwind_1,'lcc',inverse=False) -uwind_2, vwind_2 = rotate_wind(Lat0,Lon0,lon2,uwind_2,vwind_2,'lcc',inverse=False) -wspd10m_1 = np.sqrt(uwind_1**2 + vwind_1**2) -wspd10m_2 = np.sqrt(uwind_2**2 + vwind_2**2) -wspd10m_diff = wspd10m_2 - wspd10m_1 + uwind_1, vwind_1 = rotate_wind(Lat0,Lon0,lon,uwind_1,vwind_1,'lcc',inverse=False) + uwind_2, vwind_2 = rotate_wind(Lat0,Lon0,lon2,uwind_2,vwind_2,'lcc',inverse=False) + wspd10m_1 = np.sqrt(uwind_1**2 + vwind_1**2) + wspd10m_2 = np.sqrt(uwind_2**2 + vwind_2**2) + wspd10m_diff = wspd10m_2 - wspd10m_1 # Surface-based CAPE -cape_1 = data1.select(name='Convective available potential energy',typeOfLevel='surface')[0].values -cape_2 = data2.select(name='Convective available potential energy',typeOfLevel='surface')[0].values -cape_diff = cape_2 - cape_1 + cape_1 = data1.select(name='Convective available potential energy',typeOfLevel='surface')[0].values + cape_2 = data2.select(name='Convective available potential energy',typeOfLevel='surface')[0].values + cape_diff = cape_2 - cape_1 # Surface-based CIN -cin_1 = data1.select(name='Convective inhibition',typeOfLevel='surface')[0].values -cin_2 = data2.select(name='Convective inhibition',typeOfLevel='surface')[0].values -cin_diff = cin_2 - cin_1 + cin_1 = data1.select(name='Convective inhibition',typeOfLevel='surface')[0].values + cin_2 = data2.select(name='Convective inhibition',typeOfLevel='surface')[0].values + cin_diff = cin_2 - cin_1 # 500 mb height, wind, vorticity -z500_1 = data1.select(name='Geopotential Height',level=500)[0].values * 0.1 -z500_1 = ndimage.filters.gaussian_filter(z500_1, 6.89) -z500_2 = data2.select(name='Geopotential Height',level=500)[0].values * 0.1 -z500_2 = ndimage.filters.gaussian_filter(z500_2, 6.89) -z500_diff = z500_2 - z500_1 -vort500_1 = data1.select(name='Absolute vorticity',level=500)[0].values * 100000 -vort500_1 = ndimage.filters.gaussian_filter(vort500_1,1.7225) -vort500_1[vort500_1 > 1000] = 0 # Mask out undefined values on domain edge -vort500_2 = data2.select(name='Absolute vorticity',level=500)[0].values * 100000 -vort500_2 = ndimage.filters.gaussian_filter(vort500_2,1.7225) -vort500_2[vort500_2 > 1000] = 0 # Mask out undefined values on domain edge -u500_1 = data1.select(name='U component of wind',level=500)[0].values * 1.94384 -u500_2 = data2.select(name='U component of wind',level=500)[0].values * 1.94384 -v500_1 = data1.select(name='V component of wind',level=500)[0].values * 1.94384 -v500_2 = data2.select(name='V component of wind',level=500)[0].values * 1.94384 + z500_1 = data1.select(name='Geopotential Height',level=500)[0].values * 0.1 + z500_1 = ndimage.filters.gaussian_filter(z500_1, 6.89) + z500_2 = data2.select(name='Geopotential Height',level=500)[0].values * 0.1 + z500_2 = ndimage.filters.gaussian_filter(z500_2, 6.89) + z500_diff = z500_2 - z500_1 + vort500_1 = data1.select(name='Absolute vorticity',level=500)[0].values * 100000 + vort500_1 = ndimage.filters.gaussian_filter(vort500_1,1.7225) + vort500_1[vort500_1 > 1000] = 0 # Mask out undefined values on domain edge + vort500_2 = data2.select(name='Absolute vorticity',level=500)[0].values * 100000 + vort500_2 = ndimage.filters.gaussian_filter(vort500_2,1.7225) + vort500_2[vort500_2 > 1000] = 0 # Mask out undefined values on domain edge + u500_1 = data1.select(name='U component of wind',level=500)[0].values * 1.94384 + u500_2 = data2.select(name='U component of wind',level=500)[0].values * 1.94384 + v500_1 = data1.select(name='V component of wind',level=500)[0].values * 1.94384 + v500_2 = data2.select(name='V component of wind',level=500)[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -u500_1, v500_1 = rotate_wind(Lat0,Lon0,lon,u500_1,v500_1,'lcc',inverse=False) -u500_2, v500_2 = rotate_wind(Lat0,Lon0,lon2,u500_2,v500_2,'lcc',inverse=False) + u500_1, v500_1 = rotate_wind(Lat0,Lon0,lon,u500_1,v500_1,'lcc',inverse=False) + u500_2, v500_2 = rotate_wind(Lat0,Lon0,lon2,u500_2,v500_2,'lcc',inverse=False) # 250 mb winds -u250_1 = data1.select(name='U component of wind',level=250)[0].values * 1.94384 -u250_2 = data2.select(name='U component of wind',level=250)[0].values * 1.94384 -v250_1 = data1.select(name='V component of wind',level=250)[0].values * 1.94384 -v250_2 = data2.select(name='V component of wind',level=250)[0].values * 1.94384 + u250_1 = data1.select(name='U component of wind',level=250)[0].values * 1.94384 + u250_2 = data2.select(name='U component of wind',level=250)[0].values * 1.94384 + v250_1 = data1.select(name='V component of wind',level=250)[0].values * 1.94384 + v250_2 = data2.select(name='V component of wind',level=250)[0].values * 1.94384 # Rotate winds from grid relative to Earth relative -u250_1, v250_1 = rotate_wind(Lat0,Lon0,lon,u250_1,v250_1,'lcc',inverse=False) -u250_2, v250_2 = rotate_wind(Lat0,Lon0,lon2,u250_2,v250_2,'lcc',inverse=False) -wspd250_1 = np.sqrt(u250_1**2 + v250_1**2) -wspd250_2 = np.sqrt(u250_2**2 + v250_2**2) -wspd250_diff = wspd250_2 - wspd250_1 + u250_1, v250_1 = rotate_wind(Lat0,Lon0,lon,u250_1,v250_1,'lcc',inverse=False) + u250_2, v250_2 = rotate_wind(Lat0,Lon0,lon2,u250_2,v250_2,'lcc',inverse=False) + wspd250_1 = np.sqrt(u250_1**2 + v250_1**2) + wspd250_2 = np.sqrt(u250_2**2 + v250_2**2) + wspd250_diff = wspd250_2 - wspd250_1 # Total precipitation -qpf_1 = data1.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 -qpf_2 = data2.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 -qpf_diff = qpf_2 - qpf_1 + qpf_1 = data1.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 + qpf_2 = data2.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 + qpf_diff = qpf_2 - qpf_1 # Composite reflectivity -refc_1 = data1.select(name='Maximum/Composite radar reflectivity')[0].values -refc_2 = data2.select(name='Maximum/Composite radar reflectivity')[0].values + refc_1 = data1.select(name='Maximum/Composite radar reflectivity')[0].values + refc_2 = data2.select(name='Maximum/Composite radar reflectivity')[0].values -if (fhr > 0): + if (fhr > 0): # Max/Min Hourly 2-5 km Updraft Helicity - maxuh25_1 = data1.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values - maxuh25_2 = data2.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values - minuh25_1 = data1.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values - minuh25_2 = data2.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values - maxuh25_1[maxuh25_1 < 10] = 0 - maxuh25_2[maxuh25_2 < 10] = 0 - minuh25_1[minuh25_1 > -10] = 0 - minuh25_2[minuh25_2 > -10] = 0 - uh25_1 = maxuh25_1 + minuh25_1 - uh25_2 = maxuh25_2 + minuh25_2 - uh25_diff = uh25_2 - uh25_1 + maxuh25_1 = data1.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values + maxuh25_2 = data2.select(stepType='max',parameterName="199",topLevel=5000,bottomLevel=2000)[0].values + minuh25_1 = data1.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values + minuh25_2 = data2.select(stepType='min',parameterName="200",topLevel=5000,bottomLevel=2000)[0].values + maxuh25_1[maxuh25_1 < 10] = 0 + maxuh25_2[maxuh25_2 < 10] = 0 + minuh25_1[minuh25_1 > -10] = 0 + minuh25_2[minuh25_2 > -10] = 0 + uh25_1 = maxuh25_1 + minuh25_1 + uh25_2 = maxuh25_2 + minuh25_2 + uh25_diff = uh25_2 - uh25_1 -t2a = time.perf_counter() -t3a = round(t2a-t1a, 3) -print(("%.3f seconds to read all messages") % t3a) + t2a = time.perf_counter() + t3a = round(t2a-t1a, 3) + print(("%.3f seconds to read all messages") % t3a) ######################################## # START PLOTTING FOR EACH DOMAIN # ######################################## -def main(): + def main(): # Number of processes must coincide with the number of domains to plot - pool = multiprocessing.Pool(len(domains)) - pool.map(plot_all,domains) + pool = multiprocessing.Pool(len(domains)) + pool.map(plot_all,domains) -def plot_all(dom): + def plot_all(dom): - t1dom = time.perf_counter() - print(('Working on '+dom)) + t1dom = time.perf_counter() # Map corners for each domain - if dom == 'conus': - llcrnrlon = -120.5 - llcrnrlat = 21.0 - urcrnrlon = -64.5 - urcrnrlat = 49.0 - lat_0 = 35.4 - lon_0 = -97.6 - extent=[llcrnrlon-3,urcrnrlon-6,llcrnrlat-1,urcrnrlat+2] - elif dom == 'regional': - llcrnrlon = np.min(lon) - llcrnrlat = np.min(lat) - urcrnrlon = np.max(lon) - urcrnrlat = np.max(lat) - lat_0 = Lat0 - lon_0 = Lon0 - extent=[llcrnrlon,urcrnrlon,llcrnrlat-1,urcrnrlat] + if dom == 'conus': + llcrnrlon = -120.5 + llcrnrlat = 21.0 + urcrnrlon = -64.5 + urcrnrlat = 49.0 + lat_0 = 35.4 + lon_0 = -97.6 + extent=[llcrnrlon-3,urcrnrlon-6,llcrnrlat-1,urcrnrlat+2] + elif dom == 'regional': + llcrnrlon = np.min(lon) + llcrnrlat = np.min(lat) + urcrnrlon = np.max(lon) + urcrnrlat = np.max(lat) + lat_0 = Lat0 + lon_0 = Lon0 + extent=[llcrnrlon,urcrnrlon,llcrnrlat-1,urcrnrlat] # create figure and axes instances - fig = plt.figure(figsize=(10,10)) - gs = GridSpec(9,9,wspace=0.0,hspace=0.0) + fig = plt.figure(figsize=(10,10)) + gs = GridSpec(9,9,wspace=0.0,hspace=0.0) # Define where Cartopy Maps are located - cartopy.config['data_dir'] = CARTOPY_DIR + cartopy.config['data_dir'] = CARTOPY_DIR - back_res='50m' - back_img='on' + back_res='50m' + back_img='on' # set up the map background with cartopy - myproj=ccrs.LambertConformal(central_longitude=lon_0, central_latitude=lat_0, false_easting=0.0, - false_northing=0.0, secant_latitudes=None, standard_parallels=None, - globe=None) - ax1 = fig.add_subplot(gs[0:4,0:4], projection=myproj) - ax2 = fig.add_subplot(gs[0:4,5:], projection=myproj) - ax3 = fig.add_subplot(gs[5:,1:8], projection=myproj) - ax1.set_extent(extent) - ax2.set_extent(extent) - ax3.set_extent(extent) - - fline_wd = 0.5 # line width - falpha = 0.3 # transparency + myproj=ccrs.LambertConformal(central_longitude=lon_0, central_latitude=lat_0, false_easting=0.0, + false_northing=0.0, secant_latitudes=None, standard_parallels=None, + globe=None) + ax1 = fig.add_subplot(gs[0:4,0:4], projection=myproj) + ax2 = fig.add_subplot(gs[0:4,5:], projection=myproj) + ax3 = fig.add_subplot(gs[5:,1:8], projection=myproj) + ax1.set_extent(extent) + ax2.set_extent(extent) + ax3.set_extent(extent) + + fline_wd = 0.5 # line width + falpha = 0.3 # transparency # natural_earth # land=cfeature.NaturalEarthFeature('physical','land',back_res, # edgecolor='face',facecolor=cfeature.COLORS['land'], # alpha=falpha) - lakes=cfeature.NaturalEarthFeature('physical','lakes',back_res, - edgecolor='blue',facecolor='none', - linewidth=fline_wd,alpha=falpha) - coastline=cfeature.NaturalEarthFeature('physical','coastline', - back_res,edgecolor='blue',facecolor='none', - linewidth=fline_wd,alpha=falpha) - states=cfeature.NaturalEarthFeature('cultural','admin_1_states_provinces', - back_res,edgecolor='black',facecolor='none', - linewidth=fline_wd,linestyle=':',alpha=falpha) - borders=cfeature.NaturalEarthFeature('cultural','admin_0_countries', - back_res,edgecolor='red',facecolor='none', - linewidth=fline_wd,alpha=falpha) + lakes=cfeature.NaturalEarthFeature('physical','lakes',back_res, + edgecolor='blue',facecolor='none', + linewidth=fline_wd,alpha=falpha) + coastline=cfeature.NaturalEarthFeature('physical','coastline', + back_res,edgecolor='blue',facecolor='none', + linewidth=fline_wd,alpha=falpha) + states=cfeature.NaturalEarthFeature('cultural','admin_1_states_provinces', + back_res,edgecolor='black',facecolor='none', + linewidth=fline_wd,linestyle=':',alpha=falpha) + borders=cfeature.NaturalEarthFeature('cultural','admin_0_countries', + back_res,edgecolor='red',facecolor='none', + linewidth=fline_wd,alpha=falpha) # All lat lons are earth relative, so setup the associated projection correct for that data - transform = ccrs.PlateCarree() + transform = ccrs.PlateCarree() # high-resolution background images - if back_img=='on': - img = plt.imread(CARTOPY_DIR+'/raster_files/NE1_50M_SR_W.tif') - ax1.imshow(img, origin='upper', transform=transform) - ax2.imshow(img, origin='upper', transform=transform) - ax3.imshow(img, origin='upper', transform=transform) + if back_img=='on': + img = plt.imread(CARTOPY_DIR+'/raster_files/NE1_50M_SR_W.tif') + ax1.imshow(img, origin='upper', transform=transform) + ax2.imshow(img, origin='upper', transform=transform) + ax3.imshow(img, origin='upper', transform=transform) # ax.add_feature(land) - ax1.add_feature(lakes) - ax1.add_feature(states) - ax1.add_feature(borders) - ax1.add_feature(coastline) - ax2.add_feature(lakes) - ax2.add_feature(states) - ax2.add_feature(borders) - ax2.add_feature(coastline) - ax3.add_feature(lakes) - ax3.add_feature(states) - ax3.add_feature(borders) - ax3.add_feature(coastline) + ax1.add_feature(lakes) + ax1.add_feature(states) + ax1.add_feature(borders) + ax1.add_feature(coastline) + ax2.add_feature(lakes) + ax2.add_feature(states) + ax2.add_feature(borders) + ax2.add_feature(coastline) + ax3.add_feature(lakes) + ax3.add_feature(states) + ax3.add_feature(borders) + ax3.add_feature(coastline) # Map/figure has been set up here, save axes instances for use again later - keep_ax_lst_1 = ax1.get_children()[:] - keep_ax_lst_2 = ax2.get_children()[:] - keep_ax_lst_3 = ax3.get_children()[:] + keep_ax_lst_1 = ax1.get_children()[:] + keep_ax_lst_2 = ax2.get_children()[:] + keep_ax_lst_3 = ax3.get_children()[:] # colors for difference plots, only need to define once - diffcolors = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','white','white','#EEEE00','#EEC900','darkorange','orangered','red'] + diffcolors = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','white','white','#EEEE00','#EEC900','darkorange','orangered','red'] ################################ # Plot SLP ################################ - t1 = time.perf_counter() - print(('Working on slp for '+dom)) - - units = 'mb' - clevs = [976,980,984,988,992,996,1000,1004,1008,1012,1016,1020,1024,1028,1032,1036,1040,1044,1048,1052] - clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] - cm = plt.cm.Spectral_r - cmdiff = matplotlib.colors.ListedColormap(diffcolors) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs1_a = ax1.pcolormesh(lon_shift,lat_shift,slp_1,transform=transform,cmap=cm,norm=norm) - cbar1 = plt.colorbar(cs1_a,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=5) - cs1_b = ax1.contour(lon_shift,lat_shift,slpsmooth_1,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) - plt.clabel(cs1_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=6) - ax1.text(.5,1.03,'FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,slp_2,transform=transform,cmap=cm,norm=norm) - cbar2 = plt.colorbar(cs2_a,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=5) - cs2_b = ax2.contour(lon2_shift,lat2_shift,slpsmooth_2,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) - plt.clabel(cs2_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=6) - ax2.text(.5,1.03,'FV3-LAM-2 SLP ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=8,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,slp_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=5) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM SLP ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('slp_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot slp for: '+dom) % t3) + t1 = time.perf_counter() + print(('Working on slp for '+dom)) + + units = 'mb' + clevs = [976,980,984,988,992,996,1000,1004,1008,1012,1016,1020,1024,1028,1032,1036,1040,1044,1048,1052] + clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] + cm = plt.cm.Spectral_r + cmdiff = matplotlib.colors.ListedColormap(diffcolors) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs1_a = ax1.pcolormesh(lon_shift,lat_shift,slp_1,transform=transform,cmap=cm,norm=norm) + cbar1 = plt.colorbar(cs1_a,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=5) + cs1_b = ax1.contour(lon_shift,lat_shift,slpsmooth_1,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) + plt.clabel(cs1_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=6) + ax1.text(.5,1.03,'FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,slp_2,transform=transform,cmap=cm,norm=norm) + cbar2 = plt.colorbar(cs2_a,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=5) + cs2_b = ax2.contour(lon2_shift,lat2_shift,slpsmooth_2,np.arange(940,1060,4),colors='black',linewidths=1.25,transform=transform) + plt.clabel(cs2_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=6) + ax2.text(.5,1.03,'FV3-LAM-2 SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,slp_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=5) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/slp_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot slp for: '+dom) % t3) ################################# # Plot 2-m T ################################# - t1 = time.perf_counter() - print(('Working on t2m for '+dom)) + t1 = time.perf_counter() + print(('Working on t2m for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = '\xb0''F' - clevs = np.linspace(-16,134,51) - clevsdiff = [-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6] + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = '\xb0''F' + clevs = np.linspace(-16,134,51) + clevsdiff = [-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6] # cm = plt.cm.Spectral_r - cm = cmap_t2m() - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,tmp2m_1,transform=transform,cmap=cm,norm=norm) - cs_1.cmap.set_under('white') - cs_1.cmap.set_over('white') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=5) - ax1.text(.5,1.03,'FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,tmp2m_2,transform=transform,cmap=cm,norm=norm) - cs_2.cmap.set_under('white') - cs_2.cmap.set_over('white') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=5) - ax2.text(.5,1.03,'FV3-LAM-2 2-m Temperature ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,tmp2m_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('2mt_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 2mt for: '+dom) % t3) + cm = cmap_t2m() + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,tmp2m_1,transform=transform,cmap=cm,norm=norm) + cs_1.cmap.set_under('white') + cs_1.cmap.set_over('white') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=5) + ax1.text(.5,1.03,'FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,tmp2m_2,transform=transform,cmap=cm,norm=norm) + cs_2.cmap.set_under('white') + cs_2.cmap.set_over('white') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=[-16,-4,8,20,32,44,56,68,80,92,104,116,128],extend='both') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=5) + ax2.text(.5,1.03,'FV3-LAM-2 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,tmp2m_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 2-m Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/2mt_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 2mt for: '+dom) % t3) ################################# # Plot 2-m Dew Point ################################# - t1 = time.perf_counter() - print(('Working on 2mdew for '+dom)) + t1 = time.perf_counter() + print(('Working on 2mdew for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = '\xb0''F' - clevs = np.linspace(-5,80,35) - clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] - cm = cmap_q2m() - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,dew2m_1,transform=transform,cmap=cm,norm=norm) - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=6) - ax1.text(.5,1.03,'FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,dew2m_2,transform=transform,cmap=cm,norm=norm) - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=6) - ax2.text(.5,1.03,'FV3-LAM-2 2-m Dew Point Temperature ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,dew2m_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('2mdew_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 2mdew for: '+dom) % t3) + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = '\xb0''F' + clevs = np.linspace(-5,80,35) + clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] + cm = cmap_q2m() + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,dew2m_1,transform=transform,cmap=cm,norm=norm) + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=6) + ax1.text(.5,1.03,'FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,dew2m_2,transform=transform,cmap=cm,norm=norm) + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=6) + ax2.text(.5,1.03,'FV3-LAM-2 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,dew2m_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 2-m Dew Point Temperature ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/2mdew_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 2mdew for: '+dom) % t3) ################################# # Plot 10-m WSPD ################################# - t1 = time.perf_counter() - print(('Working on 10mwspd for '+dom)) + t1 = time.perf_counter() + print(('Working on 10mwspd for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = 'kts' + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = 'kts' # Places a wind barb every ~180 km, optimized for CONUS domain - skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') - barblength = 4 - - clevs = [5,10,15,20,25,30,35,40,45,50,55,60] - clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] - colorlist = ['turquoise','dodgerblue','blue','#FFF68F','#E3CF57','peru','brown','crimson','red','fuchsia','DarkViolet'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,wspd10m_1,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=6) - ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],uwind_1[::skip,::skip],vwind_1[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax1.text(.5,1.03,'FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,wspd10m_2,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('black') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=6) - ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],uwind_2[::skip,::skip],vwind_2[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax2.text(.5,1.03,'FV3-LAM-2 10-m Winds ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,wspd10m_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + skip = round(177.28*(dx/1000.)**-.97) + print('skipping every '+str(skip)+' grid points to plot') + barblength = 4 + + clevs = [5,10,15,20,25,30,35,40,45,50,55,60] + clevsdiff = [-12,-10,-8,-6,-4,-2,0,2,4,6,8,10,12] + colorlist = ['turquoise','dodgerblue','blue','#FFF68F','#E3CF57','peru','brown','crimson','red','fuchsia','DarkViolet'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,wspd10m_1,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('black') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=6) + ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],uwind_1[::skip,::skip],vwind_1[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax1.text(.5,1.03,'FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,wspd10m_2,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_2.cmap.set_under('white',alpha=0.) + cs_2.cmap.set_over('black') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=6) + ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],uwind_2[::skip,::skip],vwind_2[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax2.text(.5,1.03,'FV3-LAM-2 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,wspd10m_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('10mwind_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 10mwspd for: '+dom) % t3) + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/10mwind_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot 10mwspd for: '+dom) % t3) ################################# # Plot Surface-Based CAPE/CIN ################################# - t1 = time.perf_counter() - print(('Working on surface-based CAPE/CIN for '+dom)) + t1 = time.perf_counter() + print(('Working on surface-based CAPE/CIN for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = 'J/kg' - clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] - clevs2 = [-2000,-500,-250,-100,-25] - clevsdiff = [-2000,-1500,-1000,-500,-250,-100,0,100,250,500,1000,1500,2000] - colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,cape_1,transform=transform,cmap=cm,vmin=100,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('darkviolet') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=4) - cs_1b = ax1.contourf(lon_shift,lat_shift,cin_1,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) - ax1.text(.5,1.05,'FV3-LAM Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,cape_2,transform=transform,cmap=cm,vmin=100,norm=norm) - cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('darkviolet') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=4) - cs_2b = ax2.contourf(lon2_shift,lat2_shift,cin_2,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) - ax2.text(.5,1.05,'FV3-LAM-2 Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,cape_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM Surface-Based CAPE ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('sfcape_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot surface-based CAPE/CIN for: '+dom) % t3) - + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) -################################# - # Plot 500 mb HGT/WIND/VORT -################################# - t1 = time.perf_counter() - print(('Working on 500 mb Hgt/Wind/Vort for '+dom)) + units = 'J/kg' + clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] + clevs2 = [-2000,-500,-250,-100,-25] + clevsdiff = [-2000,-1500,-1000,-500,-250,-100,0,100,250,500,1000,1500,2000] + colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = 'x10${^5}$ s${^{-1}}$' - skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') - barblength = 4 - - vortlevs = [16,20,24,28,32,36,40] - clevsdiff = [-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6] - colorlist = ['yellow','gold','goldenrod','orange','orangered','red'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(vortlevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs1_a = ax1.pcolormesh(lon_shift,lat_shift,vort500_1,transform=transform,cmap=cm,norm=norm) - cs1_a.cmap.set_under('white') - cs1_a.cmap.set_over('darkred') - cbar1 = plt.colorbar(cs1_a,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=6) - ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u500_1[::skip,::skip],v500_1[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) - cs1_b = ax1.contour(lon_shift,lat_shift,z500_1,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) - plt.clabel(cs1_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) - ax1.text(.5,1.03,'FV3-LAM 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,vort500_2,transform=transform,cmap=cm,norm=norm) - cs2_a.cmap.set_under('white') - cs2_a.cmap.set_over('darkred') - cbar2 = plt.colorbar(cs2_a,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=6) - ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],u500_2[::skip,::skip],v500_2[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) - cs2_b = ax2.contour(lon2_shift,lat2_shift,z500_2,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) - plt.clabel(cs2_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) - ax2.text(.5,1.03,'FV3-LAM-2 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=8,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,z500_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 500-mb Heights (dam) \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('500_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 500 mb Hgt/Wind/Vort for: '+dom) % t3) + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,cape_1,transform=transform,cmap=cm,vmin=100,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('darkviolet') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=4) + cs_1b = ax1.contourf(lon_shift,lat_shift,cin_1,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) + ax1.text(.5,1.05,'FV3-LAM Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,cape_2,transform=transform,cmap=cm,vmin=100,norm=norm) + cs_2.cmap.set_under('white',alpha=0.) + cs_2.cmap.set_over('darkviolet') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=4) + cs_2b = ax2.contourf(lon2_shift,lat2_shift,cin_2,clevs2,colors='none',hatches=['**','++','////','..'],transform=transform) + ax2.text(.5,1.05,'FV3-LAM-2 Surface-Based CAPE (shaded) and CIN (hatched) ('+units+') \n <-500 (*), -500<-250 (+), -250<-100 (/), -100<-25 (.) \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) -################################# - # Plot 250 mb WIND -################################# - t1 = time.perf_counter() - print(('Working on 250 mb WIND for '+dom)) + cs = ax3.pcolormesh(lon2_shift,lat2_shift,cape_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM Surface-Based CAPE ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = 'kts' - skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') - barblength = 4 - - clevs = [50,60,70,80,90,100,110,120,130,140,150] - clevsdiff = [-30,-25,-20,-15,-10,-5,0,5,10,15,20,25,30] - colorlist = ['turquoise','deepskyblue','dodgerblue','#1874CD','blue','beige','khaki','peru','brown','crimson'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,wspd250_1,transform=transform,cmap=cm,vmin=50,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('red') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=6) - ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u250_1[::skip,::skip],v250_1[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax1.text(.5,1.03,'FV3-LAM 250 mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,wspd250_2,transform=transform,cmap=cm,vmin=50,norm=norm) - cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('red') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=6) - ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],u250_2[::skip,::skip],v250_2[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) - ax2.text(.5,1.03,'FV3-LAM-2 250 mb Winds ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs = ax3.pcolormesh(lon2_shift,lat2_shift,wspd250_diff,transform=transform,cmap=cmdiff,norm=normdiff) - cs.cmap.set_under('darkblue') - cs.cmap.set_over('darkred') - cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') - cbar3.set_label(units,fontsize=6) - cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 250-mb Winds ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('250wind_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot 250 mb WIND for: '+dom) % t3) + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/sfcape_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot surface-based CAPE/CIN for: '+dom) % t3) ################################# - # Plot Total QPF + # Plot 500 mb HGT/WIND/VORT ################################# - if (fhr > 0): # Do not make total QPF plot for forecast hour 0 t1 = time.perf_counter() - print(('Working on total qpf for '+dom)) + print(('Working on 500 mb Hgt/Wind/Vort for '+dom)) - # Clear off old plottables but keep all the map info + # Clear off old plottables but keep all the map info cbar1.remove() cbar2.remove() cbar3.remove() @@ -939,54 +814,61 @@ def plot_all(dom): clear_plotables(ax2,keep_ax_lst_2,fig) clear_plotables(ax3,keep_ax_lst_3,fig) - units = 'in' - clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] - clevsdiff = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] - colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + units = 'x10${^5}$ s${^{-1}}$' + skip = round(177.28*(dx/1000.)**-.97) + print('skipping every '+str(skip)+' grid points to plot') + barblength = 4 + + vortlevs = [16,20,24,28,32,36,40] + clevsdiff = [-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6] + colorlist = ['yellow','gold','goldenrod','orange','orangered','red'] cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + norm = matplotlib.colors.BoundaryNorm(vortlevs, cm.N) normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,qpf_1,transform=transform,cmap=cm,vmin=0.01,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('pink') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='max') + cs1_a = ax1.pcolormesh(lon_shift,lat_shift,vort500_1,transform=transform,cmap=cm,norm=norm) + cs1_a.cmap.set_under('white') + cs1_a.cmap.set_over('darkred') + cbar1 = plt.colorbar(cs1_a,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') cbar1.set_label(units,fontsize=6) - cbar1.ax.set_xticklabels([0.1,0.5,1,1.5,2,3,5,10,20]) cbar1.ax.tick_params(labelsize=6) - ax1.text(.5,1.03,'FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,qpf_2,transform=transform,cmap=cm,vmin=0.01,norm=norm) - cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('pink') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='max') + ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u500_1[::skip,::skip],v500_1[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) + cs1_b = ax1.contour(lon_shift,lat_shift,z500_1,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) + plt.clabel(cs1_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) + ax1.text(.5,1.03,'FV3-LAM 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,vort500_2,transform=transform,cmap=cm,norm=norm) + cs2_a.cmap.set_under('white') + cs2_a.cmap.set_over('darkred') + cbar2 = plt.colorbar(cs2_a,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=vortlevs,extend='both') cbar2.set_label(units,fontsize=6) - cbar2.ax.set_xticklabels([0.1,0.5,1,1.5,2,3,5,10,20]) cbar2.ax.tick_params(labelsize=6) - ax2.text(.5,1.03,'FV3-LAM-2 '+fhour2+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],u500_2[::skip,::skip],v500_2[::skip,::skip],length=barblength,linewidth=0.5,color='steelblue',transform=transform) + cs2_b = ax2.contour(lon2_shift,lat2_shift,z500_2,np.arange(486,600,6),colors='black',linewidths=1,transform=transform) + plt.clabel(cs2_b,np.arange(486,600,6),inline_spacing=1,fmt='%d',fontsize=8) + ax2.text(.5,1.03,'FV3-LAM-2 500 mb Heights (dam), Winds (kts), and $\zeta$ ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - cs = ax3.pcolormesh(lon2_shift,lat2_shift,qpf_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs = ax3.pcolormesh(lon2_shift,lat2_shift,z500_diff,transform=transform,cmap=cmdiff,norm=normdiff) cs.cmap.set_under('darkblue') cs.cmap.set_over('darkred') cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') cbar3.set_label(units,fontsize=6) cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 500-mb Heights (dam) \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('qpf_diff_'+dom+'_f'+fhour+'.png') + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/500_diff_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) - print(('%.3f seconds to plot total qpf for: '+dom) % t3) + print(('%.3f seconds to plot 500 mb Hgt/Wind/Vort for: '+dom) % t3) ################################# - # Plot Max/Min Hourly 2-5 km UH + # Plot 250 mb WIND ################################# -# Do not make max/min hourly 2-5 km UH plot for forecast hour 0 t1 = time.perf_counter() - print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) + print(('Working on 250 mb WIND for '+dom)) - # Clear off old plottables but keep all the map info + # Clear off old plottables but keep all the map info cbar1.remove() cbar2.remove() cbar3.remove() @@ -994,100 +876,214 @@ def plot_all(dom): clear_plotables(ax2,keep_ax_lst_2,fig) clear_plotables(ax3,keep_ax_lst_3,fig) - units = 'm${^2}$ s$^{-2}$' - clevs = [-150,-100,-75,-50,-25,-10,0,10,25,50,75,100,150,200,250,300] -# alternative colormap for just max UH if you don't want to plot the min UH too -# colorlist = ['white','skyblue','mediumblue','green','orchid','firebrick','#EEC900','DarkViolet'] - colorlist = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','#E5E5E5','#E5E5E5','#EEEE00','#EEC900','darkorange','orangered','red','firebrick','mediumvioletred','darkviolet'] + units = 'kts' + skip = round(177.28*(dx/1000.)**-.97) + print('skipping every '+str(skip)+' grid points to plot') + barblength = 4 + + clevs = [50,60,70,80,90,100,110,120,130,140,150] + clevsdiff = [-30,-25,-20,-15,-10,-5,0,5,10,15,20,25,30] + colorlist = ['turquoise','deepskyblue','dodgerblue','#1874CD','blue','beige','khaki','peru','brown','crimson'] cm = matplotlib.colors.ListedColormap(colorlist) - cmdiff = matplotlib.colors.ListedColormap(diffcolors) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,uh25_1,transform=transform,cmap=cm,norm=norm) - cs_1.cmap.set_under('darkblue') - cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,wspd250_1,transform=transform,cmap=cm,vmin=50,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('red') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar1.set_label(units,fontsize=6) cbar1.ax.tick_params(labelsize=6) - ax1.text(.5,1.03,'FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax1.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],u250_1[::skip,::skip],v250_1[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax1.text(.5,1.03,'FV3-LAM 250 mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,uh25_2,transform=transform,cmap=cm,norm=norm) - cs_2.cmap.set_under('darkblue') - cs_2.cmap.set_over('black') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,wspd250_2,transform=transform,cmap=cm,vmin=50,norm=norm) + cs_2.cmap.set_under('white',alpha=0.) + cs_2.cmap.set_over('red') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar2.set_label(units,fontsize=6) cbar2.ax.tick_params(labelsize=6) - ax2.text(.5,1.03,'FV3-LAM-2 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax2.barbs(lon2_shift[::skip,::skip],lat2_shift[::skip,::skip],u250_2[::skip,::skip],v250_2[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) + ax2.text(.5,1.03,'FV3-LAM-2 250 mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - cs = ax3.pcolormesh(lon2_shift,lat2_shift,uh25_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs = ax3.pcolormesh(lon2_shift,lat2_shift,wspd250_diff,transform=transform,cmap=cmdiff,norm=normdiff) cs.cmap.set_under('darkblue') cs.cmap.set_over('darkred') cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') cbar3.set_label(units,fontsize=6) cbar3.ax.tick_params(labelsize=6) - ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 250-mb Winds ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - compress_and_save('uh25_diff_'+dom+'_f'+fhour+'.png') + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/250wind_diff_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) - print(('%.3f seconds to plot Max/Min Hourly 2-5 km UH for: '+dom) % t3) + print(('%.3f seconds to plot 250 mb WIND for: '+dom) % t3) + + +################################# + # Plot Total QPF +################################# + if (fhr > 0): # Do not make total QPF plot for forecast hour 0 + t1 = time.perf_counter() + print(('Working on total qpf for '+dom)) + + # Clear off old plottables but keep all the map info + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = 'in' + clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] + clevsdiff = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] + colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,qpf_1,transform=transform,cmap=cm,vmin=0.01,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('pink') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='max') + cbar1.set_label(units,fontsize=6) + cbar1.ax.set_xticklabels([0.1,0.5,1,1.5,2,3,5,10,20]) + cbar1.ax.tick_params(labelsize=6) + ax1.text(.5,1.03,'FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,qpf_2,transform=transform,cmap=cm,vmin=0.01,norm=norm) + cs_2.cmap.set_under('white',alpha=0.) + cs_2.cmap.set_over('pink') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='max') + cbar2.set_label(units,fontsize=6) + cbar2.ax.set_xticklabels([0.1,0.5,1,1.5,2,3,5,10,20]) + cbar2.ax.tick_params(labelsize=6) + ax2.text(.5,1.03,'FV3-LAM-2 '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,qpf_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM '+fhour+'-hr Accumulated Precipitation ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/qpf_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot total qpf for: '+dom) % t3) + + +################################# + # Plot Max/Min Hourly 2-5 km UH +################################# +# Do not make max/min hourly 2-5 km UH plot for forecast hour 0 + t1 = time.perf_counter() + print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) + + # Clear off old plottables but keep all the map info + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = 'm${^2}$ s$^{-2}$' + clevs = [-150,-100,-75,-50,-25,-10,0,10,25,50,75,100,150,200,250,300] +# alternative colormap for just max UH if you don't want to plot the min UH too +# colorlist = ['white','skyblue','mediumblue','green','orchid','firebrick','#EEC900','DarkViolet'] + colorlist = ['blue','#1874CD','dodgerblue','deepskyblue','turquoise','#E5E5E5','#E5E5E5','#EEEE00','#EEC900','darkorange','orangered','red','firebrick','mediumvioletred','darkviolet'] + cm = matplotlib.colors.ListedColormap(colorlist) + cmdiff = matplotlib.colors.ListedColormap(diffcolors) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) + + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,uh25_1,transform=transform,cmap=cm,norm=norm) + cs_1.cmap.set_under('darkblue') + cs_1.cmap.set_over('black') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=6) + ax1.text(.5,1.03,'FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,uh25_2,transform=transform,cmap=cm,norm=norm) + cs_2.cmap.set_under('darkblue') + cs_2.cmap.set_over('black') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=6) + ax2.text(.5,1.03,'FV3-LAM-2 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs = ax3.pcolormesh(lon2_shift,lat2_shift,uh25_diff,transform=transform,cmap=cmdiff,norm=normdiff) + cs.cmap.set_under('darkblue') + cs.cmap.set_over('darkred') + cbar3 = plt.colorbar(cs,ax=ax3,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') + cbar3.set_label(units,fontsize=6) + cbar3.ax.tick_params(labelsize=6) + ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 1-h Max/Min 2-5 km Updraft Helicity ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/uh25_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot Max/Min Hourly 2-5 km UH for: '+dom) % t3) ################################# # Plot composite reflectivity ################################# - t1 = time.perf_counter() - print(('Working on composite reflectivity for '+dom)) + t1 = time.perf_counter() + print(('Working on composite reflectivity for '+dom)) # Clear off old plottables but keep all the map info - cbar1.remove() - cbar2.remove() - cbar3.remove() - clear_plotables(ax1,keep_ax_lst_1,fig) - clear_plotables(ax2,keep_ax_lst_2,fig) - clear_plotables(ax3,keep_ax_lst_3,fig) - - units = 'dBZ' - clevs = np.linspace(5,70,14) - clevsdiff = [20,1000] - colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] - cm = matplotlib.colors.ListedColormap(colorlist) - norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) + cbar1.remove() + cbar2.remove() + cbar3.remove() + clear_plotables(ax1,keep_ax_lst_1,fig) + clear_plotables(ax2,keep_ax_lst_2,fig) + clear_plotables(ax3,keep_ax_lst_3,fig) + + units = 'dBZ' + clevs = np.linspace(5,70,14) + clevsdiff = [20,1000] + colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] + cm = matplotlib.colors.ListedColormap(colorlist) + norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs_1 = ax1.pcolormesh(lon_shift,lat_shift,refc_1,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('black') - cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar1.set_label(units,fontsize=6) - cbar1.ax.tick_params(labelsize=6) - ax1.text(.5,1.03,'FV3-LAM Composite Reflectivity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,refc_2,transform=transform,cmap=cm,vmin=5,norm=norm) - cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('black') - cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') - cbar2.set_label(units,fontsize=6) - cbar2.ax.tick_params(labelsize=6) - ax2.text(.5,1.03,'FV3-LAM-2 Composite Reflectivity ('+units+') \n initialized: '+itime2+' valid: '+vtime2 + ' (f'+fhour2+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - csdiff = ax3.contourf(lon_shift,lat_shift,refc_1,clevsdiff,colors='red',transform=transform) - csdiff2 = ax3.contourf(lon2_shift,lat2_shift,refc_2,clevsdiff,colors='dodgerblue',transform=transform) - ax3.text(.5,1.03,'FV3-LAM (red) and FV3-LAM-2 (blue) Composite Reflectivity > 20 ('+units+') \n initialized: '+itime+' (FV3-LAM), '+itime2+' (FV3-LAM-2), valid: '+vtime+' (f'+fhour+') (FV3-LAM), '+vtime2+' (f'+fhour+') (FV3-LAM-2)',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - - compress_and_save('refc_diff_'+dom+'_f'+fhour+'.png') - t2 = time.perf_counter() - t3 = round(t2-t1, 3) - print(('%.3f seconds to plot composite reflectivity for: '+dom) % t3) + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,refc_1,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_1.cmap.set_under('white',alpha=0.) + cs_1.cmap.set_over('black') + cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar1.set_label(units,fontsize=6) + cbar1.ax.tick_params(labelsize=6) + ax1.text(.5,1.03,'FV3-LAM Composite Reflectivity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,refc_2,transform=transform,cmap=cm,vmin=5,norm=norm) + cs_2.cmap.set_under('white',alpha=0.) + cs_2.cmap.set_over('black') + cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') + cbar2.set_label(units,fontsize=6) + cbar2.ax.tick_params(labelsize=6) + ax2.text(.5,1.03,'FV3-LAM-2 Composite Reflectivity ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax2.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + csdiff = ax3.contourf(lon_shift,lat_shift,refc_1,clevsdiff,colors='red',transform=transform) + csdiff2 = ax3.contourf(lon2_shift,lat2_shift,refc_2,clevsdiff,colors='dodgerblue',transform=transform) + ax3.text(.5,1.03,'FV3-LAM (red) and FV3-LAM-2 (blue) Composite Reflectivity > 20 ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) + + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/refc_diff_'+dom+'_f'+fhour+'.png') + t2 = time.perf_counter() + t3 = round(t2-t1, 3) + print(('%.3f seconds to plot composite reflectivity for: '+dom) % t3) ###################################################### - t3dom = round(t2-t1dom, 3) - print(("%.3f seconds to plot all variables for: "+dom) % t3dom) - plt.clf() + t3dom = round(t2-t1dom, 3) + print(("%.3f seconds to plot all variables for: "+dom) % t3dom) + plt.clf() ###################################################### -main() + main() From e4354684ffa0b1da5c49f7f1e2c17b03b90740f4 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 13 Jan 2021 13:14:51 -0700 Subject: [PATCH 009/203] Make DT_ATMOS, LAYOUT_X/Y, and BLOCKSIZE user-specifiable when using predefined grids (#396) ## DESCRIPTION OF CHANGES: * When using a predefined grid, allow the default values of DT_ATMOS, LAYOUT_X, LAYOUT_Y, and BLOCKSIZE for that grid to be overwritten with values specified by the user in the experiment configuration file. * Add a WE2E test (config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh) that uses the RRFS_CONUS_25km predefined grid but changes DT_ATMOS, LAYOUT_X, LAYOUT_Y, and BLOCKSIZE from the default values for that grid. * Update comments. ## TESTS CONDUCTED: * Ran the new WE2E test (grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE) on Hera. It was successful. --- ...5km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh | 21 +++ tests/baselines_list.txt | 1 + ush/config_defaults.sh | 76 +++++---- ush/set_predef_grid_params.sh | 156 +++++++++--------- 4 files changed, 143 insertions(+), 111 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh new file mode 100644 index 0000000000..65a2322c5f --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -0,0 +1,21 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +DT_ATMOS="100" +LAYOUT_X="10" +LAYOUT_Y="4" +BLOCKSIZE="35" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GSD_SAR" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 7dc4253684..944832518a 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -32,6 +32,7 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index cd48dc2dc7..a7afb78c9a 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -734,13 +734,10 @@ ESGgrid_WIDE_HALO_WIDTH="6" # the frequency with which the top level routine in the dynamics is called # as well as the frequency with which the physics is called." # -# Note that if using one of the predefined grids (i.e. if PREDEF_GRID_NAME -# is not set to a null string, either below or in the custom workflow -# configuration file specified by EXPT_CONFIG_FN), then DT_ATMOS is -# overwritten by the value for that predefined grid. The predefined -# grid parameters are specified in the script -# -# $HOMErrfs/ush/set_predef_grid_params.sh +# Note that if using one of the predefined grids and if DT_ATMOS is not +# explicitly set in the user-specified experiment configuration file +# (EXPT_CONFIG_FN), then the default value of DT_ATMOS specified here +# will be overwritten by its default value for that predefined grid. # #----------------------------------------------------------------------- # @@ -752,6 +749,12 @@ DT_ATMOS="18" # to use in the two horizontal directions (x and y) of the regional grid # when running the forecast model. # +# Note that if using one of the predefined grids and if LAYOUT_X and/or +# LAYOUT_Y are not explicitly set in the user-specified experiment +# configuration file (EXPT_CONFIG_FN), then the default values of LAYOUT_X +# and/or LAYOUT_Y specified here will be overwritten by their default +# values for that predefined grid. +# #----------------------------------------------------------------------- # LAYOUT_X="20" @@ -759,15 +762,13 @@ LAYOUT_Y="20" # #----------------------------------------------------------------------- # -# Set BLOCKSIZE. This is the amount of data that is passed into the cache -# at a time. The number of vertical columns per MPI task needs to be -# divisible by BLOCKSIZE; otherwise, unexpected results may occur. +# Set BLOCKSIZE. This is the amount of data that is passed into the +# cache at a time. # -# GSK: IMPORTANT NOTE: -# I think Dom fixed the code so that the number of columns per MPI task -# no longer needs to be divisible by BLOCKSIZE. If so, remove the check -# on blocksize in the experiment generation scripts. Note that BLOCKSIZE -# still needs to be set to a value (probably machine-dependent). +# Note that if using one of the predefined grids and if BLOCKSIZE is not +# explicitly set in the user-specified experiment configuration file +# (EXPT_CONFIG_FN), then the default value of BLOCKSIZE specified here +# will be overwritten by its default value for that predefined grid. # #----------------------------------------------------------------------- # @@ -790,11 +791,11 @@ BLOCKSIZE="24" # # PRINT_ESMF: # Flag for whether or not to output extra (debugging) information from -# ESMF routines. Must be ".true." or ".false.". Note that the write +# ESMF routines. Must be "TRUE" or "FALSE". Note that the write # component uses ESMF library routines to interpolate from the native -# forecast model grid to the user-specified output grid (which is defined in the -# model configuration file MODEL_CONFIG_FN in the forecast's run direc- -# tory). +# forecast model grid to the user-specified output grid (which is defined +# in the model configuration file MODEL_CONFIG_FN in the forecast's run +# directory). # #----------------------------------------------------------------------- # @@ -833,21 +834,30 @@ WRTCMP_dy="" # Set PREDEF_GRID_NAME. This parameter specifies a predefined regional # grid, as follows: # -# * If PREDEF_GRID_NAME is set to an empty string, the grid generation -# method (GRID_GEN_METHOD), grid parameters, time step (DT_ATMOS), -# computational parameters (e.g. LAYOUT_X, LAYOUT_Y), and write component -# parameters set above (and possibly overwritten by values in the user- -# specified workflow configuration file) are used. -# # * If PREDEF_GRID_NAME is set to a valid predefined grid name, the grid -# generation method (GRID_GEN_METHOD), grid parameters, time step -# (DT_ATMOS), computational parameters (e.g. LAYOUT_X, LAYOUT_Y), and -# write component parameters set above (and possibly overwritten by -# values in the user-specified workflow configuration file) are overwritten -# by predefined values for the specified grid. -# -# This is simply a convenient way to quickly specify a set of parameters -# that depend on the grid. +# generation method GRID_GEN_METHOD, the (native) grid parameters, and +# the write-component grid parameters are set to predefined values for +# the specified grid, overwriting any settings of these parameters in +# the user-specified experiment configuration file. In addition, if +# the time step DT_ATMOS and the computational parameters LAYOUT_X, +# LAYOUT_Y, and BLOCKSIZE are not specified in that configuration file, +# they are also set to predefined values for the specified grid. +# +# * If PREDEF_GRID_NAME is set to an empty string, it implies the user +# is providing the native grid parameters in the user-specified +# experiment configuration file (EXPT_CONFIG_FN). In this case, the +# grid generation method GRID_GEN_METHOD, the native grid parameters, +# and the write-component grid parameters as well as the time step +# forecast model's main time step DT_ATMOS and the computational +# parameters LAYOUT_X, LAYOUT_Y, and BLOCKSIZE must be set in that +# configuration file; otherwise, the values of all of these parameters +# in this default experiment configuration file will be used. +# +# Setting PREDEF_GRID_NAME provides a convenient method of specifying a +# commonly used set of grid-dependent parameters. The predefined grid +# parameters are specified in the script +# +# $HOMErrfs/ush/set_predef_grid_params.sh # #----------------------------------------------------------------------- # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index b2ce52ed2f..371073774c 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -97,11 +97,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="300" + DT_ATMOS="${DT_ATMOS:-300}" - LAYOUT_X="5" - LAYOUT_Y="2" - BLOCKSIZE="40" + LAYOUT_X="${LAYOUT_X:-5}" + LAYOUT_Y="${LAYOUT_Y:-2}" + BLOCKSIZE="${BLOCKSIZE:-40}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -141,11 +141,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="45" + DT_ATMOS="${DT_ATMOS:-45}" - LAYOUT_X="16" - LAYOUT_Y="10" - BLOCKSIZE="32" + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-10}" + BLOCKSIZE="${BLOCKSIZE:-32}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -185,11 +185,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="40" + DT_ATMOS="${DT_ATMOS:-40}" - LAYOUT_X="30" - LAYOUT_Y="16" - BLOCKSIZE="32" + LAYOUT_X="${LAYOUT_X:-30}" + LAYOUT_Y="${LAYOUT_Y:-16}" + BLOCKSIZE="${BLOCKSIZE:-32}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -229,11 +229,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="40" + DT_ATMOS="${DT_ATMOS:-40}" - LAYOUT_X="30" - LAYOUT_Y="24" - BLOCKSIZE="35" + LAYOUT_X="${LAYOUT_X:-30}" + LAYOUT_Y="${LAYOUT_Y:-24}" + BLOCKSIZE="${BLOCKSIZE:-35}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -276,12 +276,12 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 -# DT_ATMOS="50" - DT_ATMOS="10" +# DT_ATMOS="${DT_ATMOS:-50}" + DT_ATMOS="${DT_ATMOS:-10}" - LAYOUT_X="16" - LAYOUT_Y="12" - BLOCKSIZE="40" + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-12}" + BLOCKSIZE="${BLOCKSIZE:-40}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -355,11 +355,11 @@ case ${PREDEF_GRID_NAME} in # # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" # -# DT_ATMOS="18" +# DT_ATMOS="${DT_ATMOS:-18}" # -# LAYOUT_X="24" -# LAYOUT_Y="24" -# BLOCKSIZE="15" +# LAYOUT_X="${LAYOUT_X:-24}" +# LAYOUT_Y="${LAYOUT_Y:-24}" +# BLOCKSIZE="${BLOCKSIZE:-15}" # # if [ "$QUILTING" = "TRUE" ]; then # WRTCMP_write_groups="1" @@ -392,12 +392,12 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 -# DT_ATMOS="50" - DT_ATMOS="10" +# DT_ATMOS="${DT_ATMOS:-50}" + DT_ATMOS="${DT_ATMOS:-10}" - LAYOUT_X="30" - LAYOUT_Y="17" - BLOCKSIZE="40" + LAYOUT_X="${LAYOUT_X:-30}" + LAYOUT_Y="${LAYOUT_Y:-17}" + BLOCKSIZE="${BLOCKSIZE:-40}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -450,11 +450,11 @@ case ${PREDEF_GRID_NAME} in # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" - DT_ATMOS="225" + DT_ATMOS="${DT_ATMOS:-225}" - LAYOUT_X="6" - LAYOUT_Y="4" - BLOCKSIZE="36" + LAYOUT_X="${LAYOUT_X:-6}" + LAYOUT_Y="${LAYOUT_Y:-4}" + BLOCKSIZE="${BLOCKSIZE:-36}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -505,11 +505,11 @@ case ${PREDEF_GRID_NAME} in # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" - DT_ATMOS="18" + DT_ATMOS="${DT_ATMOS:-18}" - LAYOUT_X="30" - LAYOUT_Y="22" - BLOCKSIZE="35" + LAYOUT_X="${LAYOUT_X:-30}" + LAYOUT_Y="${LAYOUT_Y:-22}" + BLOCKSIZE="${BLOCKSIZE:-35}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -590,13 +590,13 @@ case ${PREDEF_GRID_NAME} in # # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" # -# DT_ATMOS="18" +# DT_ATMOS="${DT_ATMOS:-18}" # -# LAYOUT_X="16" -# LAYOUT_Y="48" +# LAYOUT_X="${LAYOUT_X:-16}" +# LAYOUT_Y="${LAYOUT_Y:-48}" # WRTCMP_write_groups="2" # WRTCMP_write_tasks_per_group="24" -# BLOCKSIZE=32 +# BLOCKSIZE="${BLOCKSIZE:-32}" # # elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then @@ -632,17 +632,17 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="18" + DT_ATMOS="${DT_ATMOS:-18}" #Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="28" - LAYOUT_Y="16" + LAYOUT_X="${LAYOUT_X:-28}" + LAYOUT_Y="${LAYOUT_Y:-16}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="24" + BLOCKSIZE="${BLOCKSIZE:-24}" #This section is all for the write component, which you need for output during model integration if [ "$QUILTING" = "TRUE" ]; then @@ -707,16 +707,16 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="18" + DT_ATMOS="${DT_ATMOS:-18}" #Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="8" - LAYOUT_Y="8" + LAYOUT_X="${LAYOUT_X:-8}" + LAYOUT_Y="${LAYOUT_Y:-8}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="27" + BLOCKSIZE="${BLOCKSIZE:-27}" #This section is all for the write component, which you need for output during model integration if [ "$QUILTING" = "TRUE" ]; then @@ -782,17 +782,17 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="18" + DT_ATMOS="${DT_ATMOS:-18}" #Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="16" - LAYOUT_Y="8" + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-8}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="24" + BLOCKSIZE="${BLOCKSIZE:-24}" #This section is all for the write component, which you need for output during model integration if [ "$QUILTING" = "TRUE" ]; then @@ -857,16 +857,16 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="18" + DT_ATMOS="${DT_ATMOS:-18}" #Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="16" - LAYOUT_Y="12" + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-12}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="27" + BLOCKSIZE="${BLOCKSIZE:-27}" #This section is all for the write component, which you need for output during model integration if [ "$QUILTING" = "TRUE" ]; then @@ -913,11 +913,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="300" + DT_ATMOS="${DT_ATMOS:-300}" - LAYOUT_X="5" - LAYOUT_Y="5" - BLOCKSIZE="6" + LAYOUT_X="${LAYOUT_X:-5}" + LAYOUT_Y="${LAYOUT_Y:-5}" + BLOCKSIZE="${BLOCKSIZE:-6}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -955,11 +955,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="180" + DT_ATMOS="${DT_ATMOS:-180}" - LAYOUT_X="19" - LAYOUT_Y="12" - BLOCKSIZE="35" + LAYOUT_X="${LAYOUT_X:-19}" + LAYOUT_Y="${LAYOUT_Y:-12}" + BLOCKSIZE="${BLOCKSIZE:-35}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -997,11 +997,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="40" + DT_ATMOS="${DT_ATMOS:-40}" - LAYOUT_X="32" - LAYOUT_Y="24" - BLOCKSIZE="32" + LAYOUT_X="${LAYOUT_X:-32}" + LAYOUT_Y="${LAYOUT_Y:-24}" + BLOCKSIZE="${BLOCKSIZE:-32}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -1039,11 +1039,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="600" + DT_ATMOS="${DT_ATMOS:-600}" - LAYOUT_X="2" - LAYOUT_Y="3" - BLOCKSIZE="37" + LAYOUT_X="${LAYOUT_X:-2}" + LAYOUT_Y="${LAYOUT_Y:-3}" + BLOCKSIZE="${BLOCKSIZE:-37}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" @@ -1127,11 +1127,11 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH=6 - DT_ATMOS="50" + DT_ATMOS="${DT_ATMOS:-50}" - LAYOUT_X="16" - LAYOUT_Y="16" - BLOCKSIZE="30" + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-16}" + BLOCKSIZE="${BLOCKSIZE:-30}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" From 3b99ca0e66efc59e8ca352ad2d82d6ccc82bb180 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 15 Jan 2021 14:33:00 -0500 Subject: [PATCH 010/203] develop: Fix to run workflow tasks on WCOSS (#390) Fix to run workflow tasks on the WCOSS dell and cray --- .../tasks/wcoss_cray/get_extrn_ics.local | 3 ++ .../tasks/wcoss_cray/get_extrn_lbcs.local | 3 ++ modulefiles/tasks/wcoss_cray/make_grid.local | 35 ++++---------- modulefiles/tasks/wcoss_cray/make_ics.local | 35 ++++---------- modulefiles/tasks/wcoss_cray/make_lbcs.local | 35 ++++---------- modulefiles/tasks/wcoss_cray/make_orog.local | 12 +++++ .../tasks/wcoss_cray/make_sfc_climo.local | 12 +++++ modulefiles/tasks/wcoss_cray/run_fcst.local | 16 ++++++- modulefiles/tasks/wcoss_cray/run_post.local | 22 +++++---- .../tasks/wcoss_dell_p3/get_extrn_ics.local | 3 +- .../tasks/wcoss_dell_p3/get_extrn_lbcs.local | 3 +- .../tasks/wcoss_dell_p3/make_grid.local | 4 ++ .../tasks/wcoss_dell_p3/make_ics.local | 4 ++ .../tasks/wcoss_dell_p3/make_lbcs.local | 4 ++ .../tasks/wcoss_dell_p3/run_fcst.local | 4 ++ .../tasks/wcoss_dell_p3/run_post.local | 5 ++ scripts/exregional_make_orog.sh | 48 ++++--------------- scripts/exregional_run_fcst.sh | 21 +++++--- ush/launch_FV3LAM_wflow.sh | 10 ++++ ush/link_fix.sh | 2 +- ush/load_modules_run_task.sh | 4 +- ush/templates/FV3LAM_wflow.xml | 20 ++++++++ 22 files changed, 164 insertions(+), 141 deletions(-) create mode 100644 modulefiles/tasks/wcoss_cray/make_orog.local create mode 100644 modulefiles/tasks/wcoss_cray/make_sfc_climo.local diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_ics.local b/modulefiles/tasks/wcoss_cray/get_extrn_ics.local index e82aa51538..d540aea2cc 100644 --- a/modulefiles/tasks/wcoss_cray/get_extrn_ics.local +++ b/modulefiles/tasks/wcoss_cray/get_extrn_ics.local @@ -4,5 +4,8 @@ module purge +module use /usrx/local/prod/modulefiles module load hpss/4.1.0.3 +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local index 49db5c6669..d7ec3794e4 100644 --- a/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local +++ b/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local @@ -4,5 +4,8 @@ module purge +module use /usrx/local/prod/modulefiles module load hpss/4.1.0.3 +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/make_grid.local b/modulefiles/tasks/wcoss_cray/make_grid.local index 2790204cb1..edee5d7249 100644 --- a/modulefiles/tasks/wcoss_cray/make_grid.local +++ b/modulefiles/tasks/wcoss_cray/make_grid.local @@ -1,34 +1,15 @@ #%Module -module load modules -module load xt-lsfhpc -module load ncep +module use /opt/cray/ari/modulefiles module load alps -module load dvs -module load xpmem -module load ugni -module load craype-network-aries -module load switch -module load rca module load gni-headers +module load pmi/5.0.11 +module load rca module load udreg -module load hpss - -module load prod_util -module load g2tmpl-intel/1.4.0 -module load crtm-intel/2.2.6 -module load iobuf/2.0.7 -module load gempak/7.3.0 - -module load nco-gnu-sandybridge/4.4.4 -module load NetCDF-intel-sandybridge/4.2 -module load cfp-intel-sandybridge/1.1.0 -export USE_CFP=YES - -module load grib_util/1.1.0 - -module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0 +module load ugni +module load xpmem -module load python/3.6.3 +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 +module list diff --git a/modulefiles/tasks/wcoss_cray/make_ics.local b/modulefiles/tasks/wcoss_cray/make_ics.local index 2790204cb1..04eb686e14 100644 --- a/modulefiles/tasks/wcoss_cray/make_ics.local +++ b/modulefiles/tasks/wcoss_cray/make_ics.local @@ -1,34 +1,17 @@ #%Module -module load modules -module load xt-lsfhpc -module load ncep +module use /opt/cray/ari/modulefiles module load alps -module load dvs -module load xpmem -module load ugni -module load craype-network-aries -module load switch -module load rca module load gni-headers +module load pmi/5.0.11 +module load rca module load udreg -module load hpss - -module load prod_util -module load g2tmpl-intel/1.4.0 -module load crtm-intel/2.2.6 -module load iobuf/2.0.7 -module load gempak/7.3.0 - -module load nco-gnu-sandybridge/4.4.4 -module load NetCDF-intel-sandybridge/4.2 -module load cfp-intel-sandybridge/1.1.0 -export USE_CFP=YES - -module load grib_util/1.1.0 +module load ugni +module load xpmem -module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0 +module use /usrx/local/prod/modulefiles +module load hpss/4.1.0.3 -module load python/3.6.3 +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/make_lbcs.local b/modulefiles/tasks/wcoss_cray/make_lbcs.local index 2790204cb1..04eb686e14 100644 --- a/modulefiles/tasks/wcoss_cray/make_lbcs.local +++ b/modulefiles/tasks/wcoss_cray/make_lbcs.local @@ -1,34 +1,17 @@ #%Module -module load modules -module load xt-lsfhpc -module load ncep +module use /opt/cray/ari/modulefiles module load alps -module load dvs -module load xpmem -module load ugni -module load craype-network-aries -module load switch -module load rca module load gni-headers +module load pmi/5.0.11 +module load rca module load udreg -module load hpss - -module load prod_util -module load g2tmpl-intel/1.4.0 -module load crtm-intel/2.2.6 -module load iobuf/2.0.7 -module load gempak/7.3.0 - -module load nco-gnu-sandybridge/4.4.4 -module load NetCDF-intel-sandybridge/4.2 -module load cfp-intel-sandybridge/1.1.0 -export USE_CFP=YES - -module load grib_util/1.1.0 +module load ugni +module load xpmem -module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles -module load esmf/8.0.0 +module use /usrx/local/prod/modulefiles +module load hpss/4.1.0.3 -module load python/3.6.3 +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/make_orog.local b/modulefiles/tasks/wcoss_cray/make_orog.local new file mode 100644 index 0000000000..c431755327 --- /dev/null +++ b/modulefiles/tasks/wcoss_cray/make_orog.local @@ -0,0 +1,12 @@ +#%Module + +module use /opt/cray/ari/modulefiles +module load alps +module load gni-headers +module load pmi/5.0.11 +module load rca +module load udreg +module load ugni +module load xpmem + +module list diff --git a/modulefiles/tasks/wcoss_cray/make_sfc_climo.local b/modulefiles/tasks/wcoss_cray/make_sfc_climo.local new file mode 100644 index 0000000000..c431755327 --- /dev/null +++ b/modulefiles/tasks/wcoss_cray/make_sfc_climo.local @@ -0,0 +1,12 @@ +#%Module + +module use /opt/cray/ari/modulefiles +module load alps +module load gni-headers +module load pmi/5.0.11 +module load rca +module load udreg +module load ugni +module load xpmem + +module list diff --git a/modulefiles/tasks/wcoss_cray/run_fcst.local b/modulefiles/tasks/wcoss_cray/run_fcst.local index 525e87aae6..edee5d7249 100644 --- a/modulefiles/tasks/wcoss_cray/run_fcst.local +++ b/modulefiles/tasks/wcoss_cray/run_fcst.local @@ -1,3 +1,15 @@ #%Module -module unload python/2.7.14 -module load python/3.6.3 + +module use /opt/cray/ari/modulefiles +module load alps +module load gni-headers +module load pmi/5.0.11 +module load rca +module load udreg +module load ugni +module load xpmem + +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 + +module list diff --git a/modulefiles/tasks/wcoss_cray/run_post.local b/modulefiles/tasks/wcoss_cray/run_post.local index 5d1e72c237..4529dd2f37 100644 --- a/modulefiles/tasks/wcoss_cray/run_post.local +++ b/modulefiles/tasks/wcoss_cray/run_post.local @@ -1,14 +1,18 @@ #%Module -module load modules -module load xt-lsfhpc -module load ncep +module use /opt/cray/ari/modulefiles module load alps -module load dvs -module load xpmem -module load ugni -module load craype-network-aries -module load switch -module load rca module load gni-headers +module load pmi/5.0.11 +module load rca module load udreg +module load ugni +module load xpmem + +module use /usrx/local/prod/modulefiles +module load hpss/4.1.0.3 + +module use /usrx/local/nceplibs/modulefiles +module load srw-app-python/1.0.0 + + diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local index f0c321901a..1502c04d56 100644 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local +++ b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local @@ -3,6 +3,7 @@ ############################################################# module purge - module load HPSS/5.0.2.5 +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local index 8211cb9364..74e8f878c1 100644 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local +++ b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local @@ -3,6 +3,7 @@ ############################################################# module purge - module load HPSS/5.0.2.5 +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_grid.local b/modulefiles/tasks/wcoss_dell_p3/make_grid.local index 7f651f92ea..0e25550649 100644 --- a/modulefiles/tasks/wcoss_dell_p3/make_grid.local +++ b/modulefiles/tasks/wcoss_dell_p3/make_grid.local @@ -1,3 +1,7 @@ #%Module + module load lsf/10.1 module load python/3.6.3 + +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_ics.local b/modulefiles/tasks/wcoss_dell_p3/make_ics.local index 7f651f92ea..0e25550649 100644 --- a/modulefiles/tasks/wcoss_dell_p3/make_ics.local +++ b/modulefiles/tasks/wcoss_dell_p3/make_ics.local @@ -1,3 +1,7 @@ #%Module + module load lsf/10.1 module load python/3.6.3 + +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local index 7f651f92ea..0e25550649 100644 --- a/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local +++ b/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local @@ -1,3 +1,7 @@ #%Module + module load lsf/10.1 module load python/3.6.3 + +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local index 525e87aae6..5b62d935fd 100644 --- a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local +++ b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local @@ -1,3 +1,7 @@ #%Module + module unload python/2.7.14 module load python/3.6.3 + +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_post.local b/modulefiles/tasks/wcoss_dell_p3/run_post.local index 529df93fd9..9e6f62305a 100644 --- a/modulefiles/tasks/wcoss_dell_p3/run_post.local +++ b/modulefiles/tasks/wcoss_dell_p3/run_post.local @@ -1,2 +1,7 @@ #%Module + module load lsf/10.1 + +module use /usrx/local/nceplibs/dev/modulefiles +module load srw-app-python/1.0.0 + diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 2d4cdd0b56..873fab97d1 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -287,50 +287,12 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" " Starting orography file generation..." -case $MACHINE in - - "WCOSS_CRAY") -# -# On WCOSS and WCOSS_C, use cfp to run multiple tiles simulatneously for -# the orography. For now, we have only one tile in the regional case, -# but in the future we will have more. First, create an input file for -# cfp. -# - ufs_utils_ushdir="${UFS_UTILS_DIR}/ush" - res="0" # What should this be set to??? - printf "%s\n" "\ -${ufs_utils_ushdir}/${orog_gen_scr} \ -$res \ -${TILE_RGNL} \ -${FIXLAM} \ -${raw_dir} \ -${UFS_UTILS_DIR} \ -${TOPO_DIR} \ -${tmp_dir}" \ - >> ${tmp_dir}/orog.file1 - aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp ${tmp_dir}/orog.file1 - rm_vrfy ${tmp_dir}/orog.file1 - ;; - - "WCOSS_DELL_P3") - ufs_utils_ushdir="${UFS_UTILS_DIR}/ush" - res="0" # What should this be set to??? - "${exec_fp}" < "${input_redirect_fn}" || \ - print_err_msg_exit "\ -Call to executable (exec_fp) that generates the raw orography file returned -with nonzero exit code: - exec_fp = \"${exec_fp}\"" - ;; - - "CHEYENNE" | "HERA" | "ORION" | "JET" | "ODIN" | "STAMPEDE") - $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ +$APRUN "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: exec_fp = \"${exec_fp}\"" - ;; -esac # # Change location to the original directory. # @@ -457,7 +419,13 @@ cp_vrfy "${raw_orog_fp}" "${filtered_orog_fp}" # filtering executable will run) with the same name as the grid file and # point it to the actual grid file specified by grid_fp. # -ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}" + +if [ "${MACHINE}" = "WCOSS_CRAY" ]; then + ln_vrfy -fs "${grid_fp}" "${filter_dir}/${grid_fn}" +else + ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}" +fi + # # Create the namelist file (in the filter_dir directory) that the orography # filtering executable will read in. diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index e1eeeee2b3..7f78b4cf64 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -92,7 +92,12 @@ case $MACHINE in "WCOSS_CRAY") ulimit -s unlimited ulimit -a - APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" + + if [ ${PE_MEMBER01} -gt 24 ];then + APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" + else + APRUN="aprun -b -j1 -n24 -N24 -d1 -cc depth" + fi ;; "WCOSS_DELL_P3") @@ -174,7 +179,7 @@ the grid and (filtered) orography files ..." cd_vrfy ${run_dir}/INPUT relative_or_null="" -if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ]; then +if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ]; then relative_or_null="--relative" fi @@ -239,7 +244,7 @@ fi relative_or_null="" -if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ]; then +if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then relative_or_null="--relative" fi @@ -345,7 +350,7 @@ static) files in the FIXam directory: run_dir = \"${run_dir}\"" relative_or_null="" -if [ "${RUN_ENVIR}" != "nco" ]; then +if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then relative_or_null="--relative" fi @@ -395,7 +400,7 @@ Creating links in the current run directory to cycle-independent model input files in the main experiment directory..." relative_or_null="" -if [ "${RUN_ENVIR}" != "nco" ]; then +if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then relative_or_null="--relative" fi @@ -436,7 +441,11 @@ cycle's (cdate) run directory (run_dir) failed: #----------------------------------------------------------------------- # if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - relative_or_null="--relative" + if [ "${MACHINE}" = "WCOSS_CRAY" ]; then + relative_or_null="" + else + relative_or_null="--relative" + fi diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" ln_vrfy -sf ${relative_or_null} ${diag_table_fp} ${run_dir} fi diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 14cb1ad19b..702dbdf936 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -96,6 +96,16 @@ if [ "$MACHINE" = "CHEYENNE" ]; then module load rocoto elif [ "$MACHINE" = "ORION" ]; then module load contrib rocoto +elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + module purge + module load lsf/10.1 + module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ + module load ruby/2.5.1 rocoto/1.2.4 +elif [ "$MACHINE" = "WCOSS_CRAY" ]; then + module purge + module load xt-lsfhpc/9.1.3 + module use -a /usrx/local/emc_rocoto/modulefiles + module load rocoto/1.2.4 else module purge module load rocoto diff --git a/ush/link_fix.sh b/ush/link_fix.sh index dd4e008c9c..651f4fbfd0 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -351,7 +351,7 @@ Please ensure that all files have the same resolution." #----------------------------------------------------------------------- # relative_or_null="" - if [ "${run_task}" = "TRUE" ]; then + if [ "${run_task}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then relative_or_null="--relative" fi diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 047f42946c..c2e7cbc9a9 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -153,8 +153,8 @@ jjob_fp="$2" #----------------------------------------------------------------------- # machine=${MACHINE,,} -env_fn="README_${machine}_${COMPILER}.txt" -env_fp="${SR_WX_APP_TOP_DIR}/docs/${env_fn}" +env_fn="build_${machine}_${COMPILER}.env" +env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" source "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 2dab388cbf..2f20b7fafb 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -133,7 +133,11 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_GRID_TN;" "&JOBSDIR;/JREGIONAL_MAKE_GRID" + {% if machine in ["WCOSS_DELL_P3", "WCOSS_CRAY"] %} + {{ nnodes_make_grid }}:ppn=1 + {% else %} {{ nnodes_make_grid }}:ppn={{ ppn_make_grid }} + {% endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; &MAKE_GRID_TN; @@ -153,7 +157,11 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_OROG_TN;" "&JOBSDIR;/JREGIONAL_MAKE_OROG" + {% if machine in ["WCOSS_DELL_P3", "WCOSS_CRAY"] %} + {{ nnodes_make_orog }}:ppn=1 + {% else %} {{ nnodes_make_orog }}:ppn={{ ppn_make_orog }} + {% endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; &MAKE_OROG_TN; @@ -214,7 +222,13 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; + {% if machine in ["WCOSS_CRAY"] %} + + {% endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_ICS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" + {% if machine in ["WCOSS_DELL_P3"] %} + 2048M-R affinity[core] + {% endif %} {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; @@ -236,7 +250,13 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; + {% if machine in ["WCOSS_CRAY"] %} + + {% endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_LBCS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" + {% if machine in ["WCOSS_DELL_P3"] %} + 2048M-R affinity[core] + {% endif %} {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; From 9339f3c689dd1014675c84f954fa40a9804e162e Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Tue, 19 Jan 2021 19:31:13 -0700 Subject: [PATCH 011/203] Bugfixes for default values of grid parameters and forecast computational parameters; bugfix for setting FXILAM_NCO_BASEDIR in WE2E tests; other improvements. (#403) --- tests/baseline_configs/config.new_ESGgrid.sh | 10 +- tests/baseline_configs/config.new_GFDLgrid.sh | 16 +- ..._USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh | 16 +- ...d_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh | 16 +- tests/run_experiments.sh | 45 +++- ush/config_defaults.sh | 181 +++++++++----- ush/set_predef_grid_params.sh | 234 +++++++++--------- ush/setup.sh | 48 ++++ 8 files changed, 349 insertions(+), 217 deletions(-) diff --git a/tests/baseline_configs/config.new_ESGgrid.sh b/tests/baseline_configs/config.new_ESGgrid.sh index 45c315172c..3588253180 100644 --- a/tests/baseline_configs/config.new_ESGgrid.sh +++ b/tests/baseline_configs/config.new_ESGgrid.sh @@ -3,16 +3,16 @@ PREEXISTING_DIR_METHOD="rename" GRID_GEN_METHOD="ESGgrid" -ESGgrid_LON_CTR=-97.5 -ESGgrid_LAT_CTR=41.25 +ESGgrid_LON_CTR="-97.5" +ESGgrid_LAT_CTR="41.25" ESGgrid_DELX="25000.0" ESGgrid_DELY="25000.0" -ESGgrid_NX=216 -ESGgrid_NY=156 +ESGgrid_NX="216" +ESGgrid_NY="156" -ESGgrid_WIDE_HALO_WIDTH=6 +ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="40" diff --git a/tests/baseline_configs/config.new_GFDLgrid.sh b/tests/baseline_configs/config.new_GFDLgrid.sh index 3cc8b620ea..27c8544ffe 100644 --- a/tests/baseline_configs/config.new_GFDLgrid.sh +++ b/tests/baseline_configs/config.new_GFDLgrid.sh @@ -3,26 +3,26 @@ PREEXISTING_DIR_METHOD="rename" GRID_GEN_METHOD="GFDLgrid" -GFDLgrid_LON_T6_CTR=-97.5 -GFDLgrid_LAT_T6_CTR=38.5 -GFDLgrid_STRETCH_FAC=1.0001 # Cannot be exactly 1.0 because then FV3 thinnks it's a global grid. This needs to be fixed in the ufs_weather_model repo. +GFDLgrid_LON_T6_CTR="-97.5" +GFDLgrid_LAT_T6_CTR="38.5" +GFDLgrid_STRETCH_FAC="1.0001" # Cannot be exactly 1.0 because then FV3 thinnks it's a global grid. This needs to be fixed in the ufs_weather_model repo. GFDLgrid_RES="96" -GFDLgrid_REFINE_RATIO=2 +GFDLgrid_REFINE_RATIO="2" -#num_margin_cells_T6_left=9 +#num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" -#num_margin_cells_T6_right=9 +#num_margin_cells_T6_right="9" #GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" -#num_margin_cells_T6_bottom=9 +#num_margin_cells_T6_bottom="9" #GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) #GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="10" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="19" -#num_margin_cells_T6_top=9 +#num_margin_cells_T6_top="9" #GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) #GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="78" diff --git a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh b/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh index f6bd589e4e..a4f0e66852 100644 --- a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh +++ b/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh @@ -3,25 +3,25 @@ PREEXISTING_DIR_METHOD="rename" GRID_GEN_METHOD="GFDLgrid" -GFDLgrid_LON_T6_CTR=-97.5 -GFDLgrid_LAT_T6_CTR=38.5 -GFDLgrid_STRETCH_FAC=1.5 +GFDLgrid_LON_T6_CTR="-97.5" +GFDLgrid_LAT_T6_CTR="38.5" +GFDLgrid_STRETCH_FAC="1.5" GFDLgrid_RES="96" -GFDLgrid_REFINE_RATIO=2 +GFDLgrid_REFINE_RATIO="2" -#num_margin_cells_T6_left=9 +#num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" -#num_margin_cells_T6_right=9 +#num_margin_cells_T6_right="9" #GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" -#num_margin_cells_T6_bottom=9 +#num_margin_cells_T6_bottom="9" #GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="10" -#num_margin_cells_T6_top=9 +#num_margin_cells_T6_top="9" #GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" diff --git a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh b/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh index ee7e3144a4..d94d0ee822 100644 --- a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh +++ b/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh @@ -3,25 +3,25 @@ PREEXISTING_DIR_METHOD="rename" GRID_GEN_METHOD="GFDLgrid" -GFDLgrid_LON_T6_CTR=-97.5 -GFDLgrid_LAT_T6_CTR=38.5 -GFDLgrid_STRETCH_FAC=1.5 +GFDLgrid_LON_T6_CTR="-97.5" +GFDLgrid_LAT_T6_CTR="38.5" +GFDLgrid_STRETCH_FAC="1.5" GFDLgrid_RES="96" -GFDLgrid_REFINE_RATIO=2 +GFDLgrid_REFINE_RATIO="2" -#num_margin_cells_T6_left=9 +#num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" -#num_margin_cells_T6_right=9 +#num_margin_cells_T6_right="9" #GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" -#num_margin_cells_T6_bottom=9 +#num_margin_cells_T6_bottom="9" #GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="10" -#num_margin_cells_T6_top=9 +#num_margin_cells_T6_top="9" #GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 46264f1e6c..47ac5b4777 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -502,6 +502,8 @@ VERBOSE=\"${VERBOSE}\"" if [ "$MACHINE" = "HERA" ]; then pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" + elif [ "$MACHINE" = "JET" ]; then + pregen_basedir="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" elif [ "$MACHINE" = "CHEYENNE" ]; then pregen_basedir="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" else @@ -603,7 +605,35 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" RUN=\"\${EXPT_SUBDIR}\" envir=\"\${EXPT_SUBDIR}\"" # -# Set COMINgfs. +# Set FIXLAM_NCO_BASEDIR. +# + if [ "$MACHINE" = "HERA" ]; then + FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" + elif [ "$MACHINE" = "JET" ]; then + FIXLAM_NCO_BASEDIR="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/FV3LAM_pregen" + elif [ "$MACHINE" = "CHEYENNE" ]; then + FIXLAM_NCO_BASEDIR="/needs/to/be/specified" + else + print_err_msg_exit "\ +The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, +orography, and surface climatology \"fixed\" files used in NCO mode are +located has not been specified for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + str=${str}" +# +# The base directory in which the pregenerated grid, orography, and surface +# climatology \"fixed\" files used in NCO mode are located. In NCO mode, +# the workflow scripts will create symlinks (in the directory specified +# by FIXLAM) to files in a subdirectory under FIXLAM_NCO_BASDEDIR, where +# the name of the subdirectory is the name of the predefined grid specified +# by PREDEF_GRID_NAME. +# +FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" +# +# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model +# for ICs or LBCs. # if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ] || \ @@ -612,13 +642,10 @@ envir=\"\${EXPT_SUBDIR}\"" if [ "$MACHINE" = "HERA" ]; then COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" - FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" - FIXLAM_NCO_BASEDIR="/needs/to/be/specified" elif [ "$MACHINE" = "CHEYENNE" ]; then COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" - FIXLAM_NCO_BASEDIR="/needs/to/be/specified" else print_err_msg_exit "\ The directory (COMINgfs) that needs to be specified when running the @@ -634,15 +661,7 @@ for this machine (MACHINE): # mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as # the external model for ICs and/or LBCs. # -COMINgfs=\"${COMINgfs}\" -# -# The base directory in which the pregenerated grid, orography, and surface -# climatology \"fixed\" files used in NCO mode are located. In NCO mode, -# the workflow scripts will create a symlink (at the location specified -# by FIXLAM) to a subdirectory under FIXLAM_NCO_BASDEDIR. (The name of -# the subdirectory is the name of the grid specified by PREDEF_GRID_NAME.) -# -FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" +COMINgfs=\"${COMINgfs}\"" fi # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index a7afb78c9a..0234ef47a6 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -429,10 +429,12 @@ FV3GFS_FILE_FMT_LBCS="nemsio" # Set NOMADS online data associated parameters. Definitions: # # NOMADS: -# Flag controlling whether or not using NOMADS online data +# Flag controlling whether or not using NOMADS online data. # -# NOMADS_file_type -# Flag controlling the format of data +# NOMADS_file_type: +# Flag controlling the format of data. +# +#----------------------------------------------------------------------- # NOMADS="FALSE" NOMADS_file_type="nemsio" @@ -510,13 +512,26 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" # This will generate a regional grid using the map projection developed # by Jim Purser of EMC. # -# Note that if using a predefined grid (PREDEDF_GRID_NAME set to a valid -# non-empty value), this parameter is overwritten by the method used to -# generate that grid. +# Note that: +# +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then GRID_GEN_METHOD will be reset to the value of +# GRID_GEN_METHOD for that grid. This will happen regardless of +# whether or not GRID_GEN_METHOD is assigned a value in the user- +# specified experiment configuration file, i.e. any value it may be +# assigned in the experiment configuration file will be overwritten. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then GRID_GEN_METHOD must +# be set in the experiment configuration file. Otherwise, it will +# remain set to a null string, and the experiment generation will +# fail because the generation scripts check to ensure that it is set +# to a non-empty string before creating the experiment directory. # #----------------------------------------------------------------------- # -GRID_GEN_METHOD="ESGgrid" +GRID_GEN_METHOD="" # #----------------------------------------------------------------------- # @@ -656,18 +671,46 @@ GRID_GEN_METHOD="ESGgrid" # in the file names, so we allow for that here by setting this flag to # "TRUE". # +# Note that: +# +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then: +# +# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then +# these parameters will get reset to the values for that grid. +# This will happen regardless of whether or not they are assigned +# values in the user-specified experiment configuration file, i.e. +# any values they may be assigned in the experiment configuration +# file will be overwritten. +# +# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then +# these parameters will not be used and thus do not need to be reset +# to non-empty strings. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then: +# +# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified +# experiment configuration file, then these parameters must be set +# in that configuration file. +# +# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified +# experiment configuration file, then these parameters will not be +# used and thus do not need to be reset to non-empty strings. +# #----------------------------------------------------------------------- # -GFDLgrid_LON_T6_CTR=-97.5 -GFDLgrid_LAT_T6_CTR=35.5 -GFDLgrid_RES="384" -GFDLgrid_STRETCH_FAC=1.5 -GFDLgrid_REFINE_RATIO=3 -GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=10 -GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=374 -GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=10 -GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=374 -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" +GFDLgrid_LON_T6_CTR="" +GFDLgrid_LAT_T6_CTR="" +GFDLgrid_RES="" +GFDLgrid_STRETCH_FAC="" +GFDLgrid_REFINE_RATIO="" +GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="" +GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="" +GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="" +GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="" +GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="" # #----------------------------------------------------------------------- # @@ -711,68 +754,90 @@ GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" # 4-cell-wide halos that we will eventually end up with. Note that the # grid and orography files with the wide halo are only needed as intermediates # in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; -# they are not needed by the forecast model. Usually, there is no reason -# to change this parameter from its default value set here. +# they are not needed by the forecast model. +# NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified +# variable. Just set it in the function set_gridparams_ESGgrid.sh. # -# NOTE: Probably don't need to make this a user-specified variable. -# Just set it in the function set_gridparams_ESGgrid.sh. +# Note that: # -#----------------------------------------------------------------------- +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then: # -ESGgrid_LON_CTR="-97.5" -ESGgrid_LAT_CTR="35.5" -ESGgrid_DELX="3000.0" -ESGgrid_DELY="3000.0" -ESGgrid_NX="1000" -ESGgrid_NY="1000" -ESGgrid_WIDE_HALO_WIDTH="6" +# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then +# these parameters will not be used and thus do not need to be reset +# to non-empty strings. # -#----------------------------------------------------------------------- +# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then +# these parameters will get reset to the values for that grid. +# This will happen regardless of whether or not they are assigned +# values in the user-specified experiment configuration file, i.e. +# any values they may be assigned in the experiment configuration +# file will be overwritten. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then: # -# Set DT_ATMOS. This is the main forecast model integraton time step. -# As described in the forecast model documentation, "It corresponds to -# the frequency with which the top level routine in the dynamics is called -# as well as the frequency with which the physics is called." +# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified +# experiment configuration file, then these parameters will not be +# used and thus do not need to be reset to non-empty strings. # -# Note that if using one of the predefined grids and if DT_ATMOS is not -# explicitly set in the user-specified experiment configuration file -# (EXPT_CONFIG_FN), then the default value of DT_ATMOS specified here -# will be overwritten by its default value for that predefined grid. +# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified +# experiment configuration file, then these parameters must be set +# in that configuration file. # #----------------------------------------------------------------------- # -DT_ATMOS="18" +ESGgrid_LON_CTR="" +ESGgrid_LAT_CTR="" +ESGgrid_DELX="" +ESGgrid_DELY="" +ESGgrid_NX="" +ESGgrid_NY="" +ESGgrid_WIDE_HALO_WIDTH="" # #----------------------------------------------------------------------- # -# Set LAYOUT_X and LAYOUT_Y. These are the number of MPI tasks (processes) -# to use in the two horizontal directions (x and y) of the regional grid -# when running the forecast model. +# Set computational parameters for the forecast. Definitions: # -# Note that if using one of the predefined grids and if LAYOUT_X and/or -# LAYOUT_Y are not explicitly set in the user-specified experiment -# configuration file (EXPT_CONFIG_FN), then the default values of LAYOUT_X -# and/or LAYOUT_Y specified here will be overwritten by their default -# values for that predefined grid. +# DT_ATMOS: +# The main forecast model integraton time step. As described in the +# forecast model documentation, "It corresponds to the frequency with +# which the top level routine in the dynamics is called as well as the +# frequency with which the physics is called." # -#----------------------------------------------------------------------- +# LAYOUT_X, LAYOUT_Y: +# The number of MPI tasks (processes) to use in the two horizontal +# directions (x and y) of the regional grid when running the forecast +# model. # -LAYOUT_X="20" -LAYOUT_Y="20" +# BLOCKSIZE: +# The amount of data that is passed into the cache at a time. # -#----------------------------------------------------------------------- +# Here, we set these parameters to null strings. This is so that, for +# any one of these parameters: # -# Set BLOCKSIZE. This is the amount of data that is passed into the -# cache at a time. +# 1) If the experiment is using a predefined grid, then if the user +# sets the parameter in the user-specified experiment configuration +# file (EXPT_CONFIG_FN), that value will be used in the forecast(s). +# Otherwise, the default value of the parameter for that predefined +# grid will be used. # -# Note that if using one of the predefined grids and if BLOCKSIZE is not -# explicitly set in the user-specified experiment configuration file -# (EXPT_CONFIG_FN), then the default value of BLOCKSIZE specified here -# will be overwritten by its default value for that predefined grid. +# 2) If the experiment is not using a predefined grid (i.e. it is using +# a custom grid whose parameters are specified in the experiment +# configuration file), then the user must specify a value for the +# parameter in that configuration file. Otherwise, the parameter +# will remain set to a null string, and the experiment generation +# will fail because the generation scripts check to ensure that all +# the parameters defined in this section are set to non-empty strings +# before creating the experiment directory. # #----------------------------------------------------------------------- # -BLOCKSIZE="24" +DT_ATMOS="" +LAYOUT_X="" +LAYOUT_Y="" +BLOCKSIZE="" # #----------------------------------------------------------------------- # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 371073774c..e87e9ecdb5 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -86,16 +86,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-97.5 - ESGgrid_LAT_CTR=38.5 + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" ESGgrid_DELX="25000.0" ESGgrid_DELY="25000.0" - ESGgrid_NX=202 - ESGgrid_NY=116 + ESGgrid_NX="202" + ESGgrid_NY="116" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-300}" @@ -130,16 +130,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-97.5 - ESGgrid_LAT_CTR=38.5 + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX=396 - ESGgrid_NY=232 + ESGgrid_NX="396" + ESGgrid_NY="232" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-45}" @@ -174,16 +174,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-97.5 - ESGgrid_LAT_CTR=38.5 + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=1748 - ESGgrid_NY=1038 + ESGgrid_NX="1748" + ESGgrid_NY="1038" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -218,16 +218,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-97.5 - ESGgrid_LAT_CTR=35.0 + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="35.0" ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=840 - ESGgrid_NY=600 + ESGgrid_NX="840" + ESGgrid_NY="600" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -265,16 +265,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-161.5 - ESGgrid_LAT_CTR=63.0 + ESGgrid_LON_CTR="-161.5" + ESGgrid_LAT_CTR="63.0" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX=320 - ESGgrid_NY=240 + ESGgrid_NX="320" + ESGgrid_NY="240" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" DT_ATMOS="${DT_ATMOS:-10}" @@ -335,22 +335,22 @@ case ${PREDEF_GRID_NAME} in # if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # -# GFDLgrid_LON_T6_CTR=-160.8 -# GFDLgrid_LAT_T6_CTR=63.0 -# GFDLgrid_STRETCH_FAC=1.161 +# GFDLgrid_LON_T6_CTR="-160.8" +# GFDLgrid_LAT_T6_CTR="63.0" +# GFDLgrid_STRETCH_FAC="1.161" # GFDLgrid_RES="768" -# GFDLgrid_REFINE_RATIO=4 +# GFDLgrid_REFINE_RATIO="4" # -# num_margin_cells_T6_left=204 +# num_margin_cells_T6_left="204" # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) # -# num_margin_cells_T6_right=204 +# num_margin_cells_T6_right="204" # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) # -# num_margin_cells_T6_bottom=249 +# num_margin_cells_T6_bottom="249" # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) # -# num_margin_cells_T6_top=249 +# num_margin_cells_T6_top="249" # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" @@ -381,16 +381,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-161.5 - ESGgrid_LAT_CTR=63.0 + ESGgrid_LON_CTR="-161.5" + ESGgrid_LAT_CTR="63.0" ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=1380 - ESGgrid_NY=1020 + ESGgrid_NX="1380" + ESGgrid_NY="1020" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" DT_ATMOS="${DT_ATMOS:-10}" @@ -430,25 +430,25 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="GFDLgrid" - GFDLgrid_LON_T6_CTR=-97.5 - GFDLgrid_LAT_T6_CTR=38.5 - GFDLgrid_STRETCH_FAC=1.4 + GFDLgrid_LON_T6_CTR="-97.5" + GFDLgrid_LAT_T6_CTR="38.5" + GFDLgrid_STRETCH_FAC="1.4" GFDLgrid_RES="96" - GFDLgrid_REFINE_RATIO=3 + GFDLgrid_REFINE_RATIO="3" - num_margin_cells_T6_left=12 + num_margin_cells_T6_left="12" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) - num_margin_cells_T6_right=12 + num_margin_cells_T6_right="12" GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) - num_margin_cells_T6_bottom=16 + num_margin_cells_T6_bottom="16" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) - num_margin_cells_T6_top=16 + num_margin_cells_T6_top="16" GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" + GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" DT_ATMOS="${DT_ATMOS:-225}" @@ -485,25 +485,25 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="GFDLgrid" - GFDLgrid_LON_T6_CTR=-97.5 - GFDLgrid_LAT_T6_CTR=38.5 - GFDLgrid_STRETCH_FAC=1.5 + GFDLgrid_LON_T6_CTR="-97.5" + GFDLgrid_LAT_T6_CTR="38.5" + GFDLgrid_STRETCH_FAC="1.5" GFDLgrid_RES="768" - GFDLgrid_REFINE_RATIO=3 + GFDLgrid_REFINE_RATIO="3" - num_margin_cells_T6_left=69 + num_margin_cells_T6_left="69" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) - num_margin_cells_T6_right=69 + num_margin_cells_T6_right="69" GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) - num_margin_cells_T6_bottom=164 + num_margin_cells_T6_bottom="164" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) - num_margin_cells_T6_top=164 + num_margin_cells_T6_top="164" GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" + GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" DT_ATMOS="${DT_ATMOS:-18}" @@ -570,22 +570,22 @@ case ${PREDEF_GRID_NAME} in #dlon=0.03 #dlat=0.03 -# GFDLgrid_LON_T6_CTR=-153.0 -# GFDLgrid_LAT_T6_CTR=61.0 -# GFDLgrid_STRETCH_FAC=1.0 # ??? +# GFDLgrid_LON_T6_CTR="-153.0" +# GFDLgrid_LAT_T6_CTR="61.0" +# GFDLgrid_STRETCH_FAC="1.0" # ??? # GFDLgrid_RES="768" -# GFDLgrid_REFINE_RATIO=3 # ??? +# GFDLgrid_REFINE_RATIO="3" # ??? # -# num_margin_cells_T6_left=61 +# num_margin_cells_T6_left="61" # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) # -# num_margin_cells_T6_right=67 +# num_margin_cells_T6_right="67" # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) # -# num_margin_cells_T6_bottom=165 +# num_margin_cells_T6_bottom="165" # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) # -# num_margin_cells_T6_top=171 +# num_margin_cells_T6_top="171" # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" @@ -606,8 +606,8 @@ case ${PREDEF_GRID_NAME} in # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR=-153.0 - ESGgrid_LAT_CTR=61.0 + ESGgrid_LON_CTR="-153.0" + ESGgrid_LAT_CTR="61.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx @@ -616,12 +616,12 @@ case ${PREDEF_GRID_NAME} in # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/ak/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX=1344 # Supergrid value 2704 - ESGgrid_NY=1152 # Supergrid value 2320 + ESGgrid_NX="1344" # Supergrid value 2704 + ESGgrid_NY="1152" # Supergrid value 2320 # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -681,8 +681,8 @@ case ${PREDEF_GRID_NAME} in # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/hi/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR=-157.0 - ESGgrid_LAT_CTR=20.0 + ESGgrid_LON_CTR="-157.0" + ESGgrid_LAT_CTR="20.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) @@ -691,12 +691,12 @@ case ${PREDEF_GRID_NAME} in # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/hi/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX=432 # Supergrid value 880 - ESGgrid_NY=360 # Supergrid value 736 + ESGgrid_NX="432" # Supergrid value 880 + ESGgrid_NY="360" # Supergrid value 736 # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -756,8 +756,8 @@ case ${PREDEF_GRID_NAME} in # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/pr/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR=-69.0 - ESGgrid_LAT_CTR=18.0 + ESGgrid_LON_CTR="-69.0" + ESGgrid_LAT_CTR="18.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) @@ -766,12 +766,12 @@ case ${PREDEF_GRID_NAME} in # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/pr/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX=576 # Supergrid value 1168 - ESGgrid_NY=432 # Supergrid value 880 + ESGgrid_NX="576" # Supergrid value 1168 + ESGgrid_NY="432" # Supergrid value 880 # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -831,8 +831,8 @@ case ${PREDEF_GRID_NAME} in # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/guam/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR=146.0 - ESGgrid_LAT_CTR=15.0 + ESGgrid_LON_CTR="146.0" + ESGgrid_LAT_CTR="15.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) @@ -841,12 +841,12 @@ case ${PREDEF_GRID_NAME} in # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/guam/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX=432 # Supergrid value 880 - ESGgrid_NY=360 # Supergrid value 736 + ESGgrid_NX="432" # Supergrid value 880 + ESGgrid_NY="360" # Supergrid value 736 # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -902,16 +902,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-62.0 - ESGgrid_LAT_CTR=22.0 + ESGgrid_LON_CTR="-62.0" + ESGgrid_LAT_CTR="22.0" ESGgrid_DELX="25000.0" ESGgrid_DELY="25000.0" - ESGgrid_NX=345 - ESGgrid_NY=230 + ESGgrid_NX="345" + ESGgrid_NY="230" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-300}" @@ -944,16 +944,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-62.0 - ESGgrid_LAT_CTR=22.0 + ESGgrid_LON_CTR="-62.0" + ESGgrid_LAT_CTR="22.0" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX=665 - ESGgrid_NY=444 + ESGgrid_NX="665" + ESGgrid_NY="444" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-180}" @@ -986,16 +986,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-62.0 - ESGgrid_LAT_CTR=22.0 + ESGgrid_LON_CTR="-62.0" + ESGgrid_LAT_CTR="22.0" ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=2880 - ESGgrid_NY=1920 + ESGgrid_NX="2880" + ESGgrid_NY="1920" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -1028,16 +1028,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-163.5 - ESGgrid_LAT_CTR=62.8 + ESGgrid_LON_CTR="-163.5" + ESGgrid_LAT_CTR="62.8" ESGgrid_DELX="50000.0" ESGgrid_DELY="50000.0" - ESGgrid_NX=74 - ESGgrid_NY=51 + ESGgrid_NX="74" + ESGgrid_NY="51" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-600}" @@ -1072,22 +1072,22 @@ case ${PREDEF_GRID_NAME} in # if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # -# GFDLgrid_LON_T6_CTR=-106.0 -# GFDLgrid_LAT_T6_CTR=54.0 -# GFDLgrid_STRETCH_FAC=0.63 +# GFDLgrid_LON_T6_CTR="-106.0" +# GFDLgrid_LAT_T6_CTR="54.0" +# GFDLgrid_STRETCH_FAC="0.63" # GFDLgrid_RES="384" -# GFDLgrid_REFINE_RATIO=3 +# GFDLgrid_REFINE_RATIO="3" # -# num_margin_cells_T6_left=10 +# num_margin_cells_T6_left="10" # GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) # -# num_margin_cells_T6_right=10 +# num_margin_cells_T6_right="10" # GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) # -# num_margin_cells_T6_bottom=10 +# num_margin_cells_T6_bottom="10" # GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) # -# num_margin_cells_T6_top=10 +# num_margin_cells_T6_top="10" # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) # # GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" @@ -1116,16 +1116,16 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-106.0 - ESGgrid_LAT_CTR=54.0 + ESGgrid_LON_CTR="-106.0" + ESGgrid_LAT_CTR="54.0" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX=960 - ESGgrid_NY=960 + ESGgrid_NX="960" + ESGgrid_NY="960" - ESGgrid_WIDE_HALO_WIDTH=6 + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-50}" diff --git a/ush/setup.sh b/ush/setup.sh index dcd2a4fe01..4a6d0b3335 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -928,6 +928,54 @@ fi # #----------------------------------------------------------------------- # +# Check to make sure that various computational parameters needed by the +# forecast model are set to non-empty values. At this point in the +# experiment generation, all of these should be set to valid (non-empty) +# values. +# +#----------------------------------------------------------------------- +# +if [ -z "${DT_ATMOS}" ]; then + print_err_msg_exit "\ +The forecast model main time step (DT_ATMOS) is set to a null string: + DT_ATMOS = ${DT_ATMOS} +Please set this to a valid numerical value in the user-specified experiment +configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" +fi + +if [ -z "${LAYOUT_X}" ]; then + print_err_msg_exit "\ +The number of MPI processes to be used in the x direction (LAYOUT_X) by +the forecast job is set to a null string: + LAYOUT_X = ${LAYOUT_X} +Please set this to a valid numerical value in the user-specified experiment +configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" +fi + +if [ -z "${LAYOUT_Y}" ]; then + print_err_msg_exit "\ +The number of MPI processes to be used in the y direction (LAYOUT_Y) by +the forecast job is set to a null string: + LAYOUT_Y = ${LAYOUT_Y} +Please set this to a valid numerical value in the user-specified experiment +configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" +fi + +if [ -z "${BLOCKSIZE}" ]; then + print_err_msg_exit "\ +The cache size to use for each MPI task of the forecast (BLOCKSIZE) is +set to a null string: + BLOCKSIZE = ${BLOCKSIZE} +Please set this to a valid numerical value in the user-specified experiment +configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" +fi +# +#----------------------------------------------------------------------- +# # If the base directory (EXPT_BASEDIR) in which the experiment subdirectory # (EXPT_SUBDIR) will be located does not start with a "/", then it is # either set to a null string or contains a relative directory. In both From 21310652e073cbe5d6fbab083fb047df87535de1 Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Fri, 22 Jan 2021 08:35:14 -0700 Subject: [PATCH 012/203] Add submission scripts for python scripts for hera specifically (#412) * Add submission scripts for python scripts for hera specifically * Updated scripts to remove hard coded paths and incorporate better practices and helpful usage statements * Typo in usage statement - there are six command line arguements --- ush/Python/sq_job.sh | 72 +++++++++++++++++++++++++++++++++++++ ush/Python/sq_job_diff.sh | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100755 ush/Python/sq_job.sh create mode 100755 ush/Python/sq_job_diff.sh diff --git a/ush/Python/sq_job.sh b/ush/Python/sq_job.sh new file mode 100755 index 0000000000..ec8317ebb9 --- /dev/null +++ b/ush/Python/sq_job.sh @@ -0,0 +1,72 @@ +#!/bin/sh + +#SBATCH --account=an_account +#SBATCH --qos=batch +#SBATCH --ntasks=4 +#SBATCH --time=0:20:00 +#SBATCH --job-name="plot_allvars" +#SBATCH --out=plot_allvars.out + +# Prior to submitting the script the following environment variables +# must be set using export or setenv +# HOMErrfs=/path-to/ufs-srweather-app/regional_workflow +# EXPTDIR=/path-to/expt_dirs/your_experiment + +cd ${HOMErrfs}/ush/Python +set -x +. /apps/lmod/lmod/init/sh + +module purge +module load hpss + +############ +# Python environment for Jet and Hera +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 +conda activate pygraf + +############ +# Python environment for Orion +############ +#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +#module load miniconda3 +#conda activate pygraf + +############ +# Python environment for Gaea +############ +#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +#module load miniconda3 +#conda activate pygraf + +############ +# Path to shape files +############ +#Hera: +SHAPE_FILES=/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/tools/NaturalEarth +#Jet: +#Orion: +#SHAPE_FILES=/home/chjeon/tools/NaturalEarth +#Gaea: + +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export FCST_START=6 +export FCST_END=${FCST_LEN_HRS} +export FCST_INC=6 + +# Usage statement: Make sure all the necessary modules can be imported. +# Six command line arguments are needed: +# 1. Cycle date/time in YYYYMMDDHH format +# 2. Starting forecast hour in HHH format +# 3. Ending forecast hour in HHH format +# 4. Forecast hour increment +# 5. EXPT_DIR: Experiment directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR/YYYYMMDDHH/postprd/ +# 6. CARTOPY_DIR: Base directory of cartopy shapefiles +# -File structure should be: +# CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp + +python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR} ${SHAPE_FILES} diff --git a/ush/Python/sq_job_diff.sh b/ush/Python/sq_job_diff.sh new file mode 100755 index 0000000000..9c49a7a1a7 --- /dev/null +++ b/ush/Python/sq_job_diff.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +#SBATCH --account=an_account +#SBATCH --qos=batch +#SBATCH --ntasks=8 +#SBATCH --time=0:30:00 +#SBATCH --job-name="plot_allvars_diff" +#SBATCH --out=plot_allvars.out + +# Prior to submitting the script the following environment variables +# must be set using export or setenv +# HOMErrfs=/path-to/ufs-srweather-app/regional_workflow +# EXPTDIR1=/path-to/expt_dirs/your_experiment1 +# EXPTDIR2=/path-to/expt_dirs/your_experiment2 + +cd ${HOMErrfs}/ush/Python +set -x +. /apps/lmod/lmod/init/sh + +module purge +module load hpss + +############ +# Python environment for Jet and Hera +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 +conda activate pygraf + +############ +# Python environment for Orion +############ +#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +#module load miniconda3 +#conda activate pygraf + +############ +# Python environment for Gaea +############ +#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +#module load miniconda3 +#conda activate pygraf + +############ +# Path to shape files +############ +#Hera: +SHAPE_FILES=/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/tools/NaturalEarth +#Jet: +#Orion: +#SHAPE_FILES=/home/chjeon/tools/NaturalEarth +#Gaea: + +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export FCST_START=6 +export FCST_END=${FCST_LEN_HRS} +export FCST_INC=3 + +# Usage statement: Make sure all the necessary modules can be imported. +# Seven command line arguments are needed: +# 1. Cycle date/time in YYYYMMDDHH format +# 2. Starting forecast hour +# 3. Ending forecast hour +# 4. Forecast hour increment +# 5. EXPT_DIR_1: Experiment 1 directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR_1/YYYYMMDDHH/postprd/ +# 6. EXPT_DIR_2: Experiment 2 directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR_2/YYYYMMDDHH/postprd/ +# 7. CARTOPY_DIR: Base directory of cartopy shapefiles +# -File structure should be: +# CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp + +python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR1} ${EXPTDIR2} ${SHAPE_FILES} From 4fae5f8b020fbb6abde200eb5fa97c886df196cc Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 25 Jan 2021 13:36:38 -0700 Subject: [PATCH 013/203] Improve the way the climatology file for Thompson MP is handled (#414) The main modification that this PR makes is that the `exregional_make_ics.sh` and `exregional_make_lbcs.sh` scripts no longer use long (and error-prone) if-statements over physics suites to set the `chgres_cube` namelist variable `thomp_mp_climo_file`; instead, the function `set_thompson_mp_fix_files.sh` that already checks whether Thompson MP is included in the SDF now also returns the new workflow variable `THOMPSON_MP_USED` that indicates whether or not Thompson MP is called by the physics suite, and the `exregional_make_ics.sh` and `exregional_make_lbcs.sh` scripts then use this flag to set the `chgres_cube` namelist variable `thomp_mp_climo_file` appropriately. The improvement being made here is that the developer no longer needs to remember which suites use Thompson MP and which don't, which must be known in order to maintain the long and repeated if-statements over physics suites currently in `exregional_make_ics.sh` and `exregional_make_lbcs.sh`. ## DESCRIPTION OF CHANGES: * In set_thompson_mp_fix_files.sh, set and output via a new function argument the variable thompson_mp_used. If the Thompson microphysics parameterization is being used by the physics suite, this will be set to "TRUE", otherwise it will be set to "FALSE". * In setup.sh: * Set the new workflow variables THOMPSON_MP_CLIMO_FN and THOMPSON_MP_CLIMO_FP. The first contains the name (but not full path) of the aerosol climatology file that may be needed by Thompson MP, and the second contains the full path of this aerosol climatology file (regardless of whether or not Thompson MP is used in the physics suite and regardless of whether or not the external model files contain aerosol data). * Set the new workflow variable THOMPSON_MP_USED (via the call to the set_thompson_mp_fix_files.sh function) that indicates whether or not Thompson MP is called by the physics suite. * Write all three new workflow variables to the experiment variable defintions file (var_defns.sh). * In exregional_make_ics.sh and exregional_make_lbcs.sh, remove suite-dependent if-statements for setting thomp_mp_climo_file. Instead, use a much simpler if-statement that sets this variable to THOMPSON_MP_CLIMO_FP only if Thompson MP is being used AND the external model does not already provide the aerosol fields needed by Thompson MP (which is all external models except the HRRR and the RAP) and otherwise leaves it set to a null string. Note that thomp_mp_climo_file is passed to the namelist file for chgres_cube. * Add a WE2E test (grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR) that uses the FV3GFS (which does not provide aerosol fields) as the external model for ICs and LBCs and uses the GSD_SAR physics suite (which includes Thompson MP). Thus, the aerosol climatology file Thompson_MP_MONTHLY_CLIMO.nc must be used to provide aerosol data to Thompson MP. This test exercises use of this aerosol climatology file in the workflow. ### Improvements: * Add a script (tests/get_expts_status.sh) that obtains the workflow status of all experiments in a specified experiments base directory. * Takes one argument -- the experiments base directory. This is the directory under which all the experiment subdirectories are located. * Assumes that all subdirectories under the base directory are experiment directories. * For each subdirectory/experiment, calls the workflow launch script to update the status of the workflow for that experiment. It then places the result from the launch script in a summary file named expts_status_${YYYYMMDDHHMN}.txt, where ${YYYYMMDDHHMN} is the 4-digit year, 2-digit month, 2-digit day-of-month, 2-digit hour-of-day, and 2-digit minute of the time at which the script was called. The summary file is placed in the experiments base directory. * For each experiment, it prints to screen the name of the experiment and the status of the corresponding workflow ("IN PROGRESS", "SUCCESS", or "FAILURE"). * Edit error messages for clarity. * Clean up comments. ## TESTS CONDUCTED: The following WE2E tests were run successfully on Hera: * grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * **grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR** * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * nco_CONUS_25km_GFDLgrid * nco_RRFS_CONUS_25km_HRRR_RAP The one that directly tests the change in the way the Thompson climatology file is handled is grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR (in bold in the list above). The remaining tests ensure pre-existing functionality is not broken. --- scripts/exregional_make_ics.sh | 152 +++++-------- scripts/exregional_make_lbcs.sh | 128 +++-------- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh | 17 ++ tests/baselines_list.txt | 1 + tests/get_expts_status.sh | 207 ++++++++++++++++++ ush/set_thompson_mp_fix_files.sh | 59 +++-- ush/setup.sh | 65 ++++-- 7 files changed, 394 insertions(+), 235 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh create mode 100755 tests/get_expts_status.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 0e0ba465a0..c81efddf2c 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -173,7 +173,8 @@ case "${CCPP_PHYS_SUITE}" in # *) print_err_msg_exit "\ -A variable mapping table has not yet been defined for this physics suite: +The variable \"varmap_file\" has not yet been specified for this physics +suite (CCPP_PHYS_SUITE): CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" ;; # @@ -223,14 +224,14 @@ esac # nsoill_out: # The number of soil layers to include in the output NetCDF file. # -# FIELD_from_climo, where FIELD = "vgtyp", "sotyp", "vgfrc", "lai", or +# FIELD_from_climo, where FIELD = "vgtyp", "sotyp", "vgfrc", "lai", or # "minmax_vgfrc": -# Logical variable indicating whether or not to obtain the field in +# Logical variable indicating whether or not to obtain the field in # question from climatology instead of the external model. The field in # question is one of vegetation type (FIELD="vgtyp"), soil type (FIELD= -# "sotyp"), vegetation fraction (FIELD="vgfrc"), leaf area index -# (FIELD="lai"), or min/max areal fractional coverage of annual green -# vegetation (FIELD="minmax_vfrr"). If FIELD_from_climo is set to +# "sotyp"), vegetation fraction (FIELD="vgfrc"), leaf area index +# (FIELD="lai"), or min/max areal fractional coverage of annual green +# vegetation (FIELD="minmax_vfrr"). If FIELD_from_climo is set to # ".true.", then the field is obtained from climatology (regardless of # whether or not it exists in an external model file). If it is set # to ".false.", then the field is obtained from the external model. @@ -300,42 +301,33 @@ minmax_vgfrc_from_climo="" lai_from_climo="" tg3_from_soil="" convert_nst="" +# +# If the physics suite uses Thompson microphysics and if the external +# model for ICs is one that does not provide the aerosol fields needed +# by Thompson microphysics (currently only the HRRR and RAP provide +# aerosol data), set the variable thomp_mp_climo_file in the chgres_cube +# namelist to the full path of the file containing aerosol climatology +# data. In this case, this file will be used to generate approximate +# aerosol fields in the ICs that Thompson MP can use. Otherwise, set +# thomp_mp_climo_file to a null string. +# thomp_mp_climo_file="" +if [ "${THOMPSON_MP_USED}" = "TRUE" ] && \ + [ "${EXTRN_MDL_NAME_ICS}" != "HRRR" -a \ + "${EXTRN_MDL_NAME_ICS}" != "RAP" ]; then + thomp_mp_climo_file="${THOMPSON_MP_CLIMO_FP}" +fi case "${EXTRN_MDL_NAME_ICS}" in - "GSMGFS") - external_model="GSMGFS" - fn_atm_nemsio="${EXTRN_MDL_FNS[0]}" fn_sfc_nemsio="${EXTRN_MDL_FNS[1]}" input_type="gfs_gaussian_nemsio" # For spectral GFS Gaussian grid in nemsio format. convert_nst=False - tracers_input="[\"spfh\",\"clwmr\",\"o3mr\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\"]" -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - else - print_err_msg_exit "\ - The chosen CCPP physics suite is unsupported at this time: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True @@ -343,54 +335,23 @@ case "${EXTRN_MDL_NAME_ICS}" in minmax_vgfrc_from_climo=True lai_from_climo=True tg3_from_soil=False - ;; - "FV3GFS") - -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - else - print_err_msg_exit "\ - The chosen CCPP physics suite is unsupported as this time: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - external_model="FV3GFS" - tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - fn_atm_nemsio="${EXTRN_MDL_FNS[0]}" fn_sfc_nemsio="${EXTRN_MDL_FNS[1]}" input_type="gaussian_nemsio" # For FV3-GFS Gaussian grid in nemsio format. convert_nst=True - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - external_model="GFS" - fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" convert_nst=False - fi - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True @@ -398,14 +359,10 @@ case "${EXTRN_MDL_NAME_ICS}" in minmax_vgfrc_from_climo=True lai_from_climo=True tg3_from_soil=False - ;; - "HRRR") - external_model="HRRR" - fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" # @@ -422,29 +379,30 @@ case "${EXTRN_MDL_NAME_ICS}" in elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then nsoill_out="9" + else + print_err_msg_exit "\ +The variable \"nsoill_out\" has not yet been specified for this external +IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" + CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" fi # # Path to the HRRRX geogrid file. # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_HRRRX" - - #Note that vgfrc, shdmin/shdmax (minmax_vgfrc), and lai fields are only available in HRRRX - #files after mid-July 2019, and only so long as the record order didn't change afterward - +# Note that vgfrc, shdmin/shdmax (minmax_vgfrc), and lai fields are only available in HRRRX +# files after mid-July 2019, and only so long as the record order didn't change afterward vgtyp_from_climo=True sotyp_from_climo=True - vgfrc_from_climo=True + vgfrc_from_climo=True minmax_vgfrc_from_climo=True lai_from_climo=True tg3_from_soil=True convert_nst=False - ;; "RAP") - external_model="RAP" - fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" # @@ -461,12 +419,17 @@ case "${EXTRN_MDL_NAME_ICS}" in elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then nsoill_out="9" + else + print_err_msg_exit "\ +The variable \"nsoill_out\" has not yet been specified for this external +IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" + CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" fi # # Path to the RAPX geogrid file. # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_RAPX" - vgtyp_from_climo=True sotyp_from_climo=False vgfrc_from_climo=True @@ -474,33 +437,12 @@ case "${EXTRN_MDL_NAME_ICS}" in lai_from_climo=True tg3_from_soil=True convert_nst=False - ;; "NAM") - external_model="NAM" - fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" - -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP -# suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - fi - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True @@ -508,14 +450,13 @@ case "${EXTRN_MDL_NAME_ICS}" in minmax_vgfrc_from_climo=True lai_from_climo=True tg3_from_soil=False - convert_nst=False - + convert_nst=False ;; *) print_err_msg_exit "\ External-model-dependent namelist variables have not yet been specified -for this external model: +for this external IC model (EXTRN_MDL_NAME_ICS): EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" ;; @@ -559,6 +500,17 @@ fi # this variable will be passed to a python script that will create the # namelist file. # +# IMPORTANT: +# If we want a namelist variable to be removed from the namelist file, +# in the "settings" variable below, we need to set its value to the +# string "null". This is equivalent to setting its value to +# !!python/none +# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the +# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. +# +# It turns out that setting the variable to an empty string also works +# to remove it from the namelist! Which is better to use?? +# settings=" 'config': { 'fix_dir_input_grid': ${FIXgsm}, @@ -584,7 +536,7 @@ settings=" 'input_type': ${input_type}, 'external_model': ${external_model}, 'tracers_input': ${tracers_input}, - 'tracers': ${tracers}, + 'tracers': ${tracers}, 'nsoill_out': $((10#${nsoill_out})), 'geogrid_file_input_grid': ${geogrid_file_input_grid}, 'vgtyp_from_climo': ${vgtyp_from_climo}, @@ -681,7 +633,7 @@ Please check the following user defined variables: FVCOM_FILE= \"${FVCOM_FILE}\" " fi - cp_vrfy ${fvcom_data_fp} ${ics_dir}/fvcom.nc + cp_vrfy ${fvcom_data_fp} ${ics_dir}/fvcom.nc cd_vrfy ${ics_dir} ${APRUN} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc || \ print_err_msg_exit "\ diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7df3ce5bef..5205b74e10 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -173,7 +173,8 @@ case "${CCPP_PHYS_SUITE}" in # *) print_err_msg_exit "\ -A variable mapping table has not yet been defined for this physics suite: +The variable \"varmap_file\" has not yet been specified for this physics +suite (CCPP_PHYS_SUITE): CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" ;; # @@ -272,135 +273,67 @@ fn_grib2="" input_type="" tracers_input="\"\"" tracers="\"\"" +# +# If the physics suite uses Thompson microphysics and if the external +# model for LBCs is one that does not provide the aerosol fields needed +# by Thompson microphysics (currently only the HRRR and RAP provide +# aerosol data), set the variable thomp_mp_climo_file in the chgres_cube +# namelist to the full path of the file containing aerosol climatology +# data. In this case, this file will be used to generate approximate +# aerosol fields in the LBCs that Thompson MP can use. Otherwise, set +# thomp_mp_climo_file to a null string. +# thomp_mp_climo_file="" +if [ "${THOMPSON_MP_USED}" = "TRUE" ] && \ + [ "${EXTRN_MDL_NAME_LBCS}" != "HRRR" -a \ + "${EXTRN_MDL_NAME_LBCS}" != "RAP" ]; then + thomp_mp_climo_file="${THOMPSON_MP_CLIMO_FP}" +fi case "${EXTRN_MDL_NAME_LBCS}" in - "GSMGFS") - external_model="GSMGFS" - input_type="gfs_gaussian_nemsio" # For spectral GFS Gaussian grid in nemsio format. - tracers_input="[\"spfh\",\"clwmr\",\"o3mr\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\"]" -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - else - print_err_msg_exit "\ - The chosen CCPP physics suite is unsupported at this time: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi - - ;; - "FV3GFS") - -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - else - print_err_msg_exit "\ - The chosen CCPP physics suite is unsupported at this time: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - external_model="FV3GFS" - input_type="gaussian_nemsio" # For FV3-GFS Gaussian grid in nemsio format. - tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - external_model="GFS" - fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" - fi - ;; - "RAP") - external_model="RAP" input_type="grib2" - ;; "HRRR") - external_model="HRRR" input_type="grib2" - ;; - "NAM") - external_model="NAM" input_type="grib2" -# -# Use Thompson climatology for ice- and water-friendly aerosols if CCPP suite uses Thompson MP -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ]; then - thomp_mp_climo_file="" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - thomp_mp_climo_file="${FIXam}/Thompson_MP_MONTHLY_CLIMO.nc" - else - print_err_msg_exit "\ - The chosen CCPP physics suite is unsupported at this time: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi - ;; *) print_err_msg_exit "\ External-model-dependent namelist variables have not yet been specified -for this external model: +for this external LBC model (EXTRN_MDL_NAME_LBCS): EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" ;; - esac # #----------------------------------------------------------------------- @@ -444,15 +377,15 @@ for (( i=0; i<${num_fhrs}; i++ )); do fn_atm_nemsio="${EXTRN_MDL_FNS[$i]}" ;; "FV3GFS") - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - fn_atm_nemsio="${EXTRN_MDL_FNS[$i]}" - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - fn_grib2="${EXTRN_MDL_FNS[$i]}" - fi + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then + fn_atm_nemsio="${EXTRN_MDL_FNS[$i]}" + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then + fn_grib2="${EXTRN_MDL_FNS[$i]}" + fi ;; "RAP") fn_grib2="${EXTRN_MDL_FNS[$i]}" - ;; + ;; "HRRR") fn_grib2="${EXTRN_MDL_FNS[$i]}" ;; @@ -462,7 +395,7 @@ for (( i=0; i<${num_fhrs}; i++ )); do *) print_err_msg_exit "\ The external model output file name to use in the chgres_cube FORTRAN name- -list file has not specified for this external model: +list file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" ;; esac @@ -495,6 +428,17 @@ list file has not specified for this external model: # this variable will be passed to a python script that will create the # namelist file. # +# IMPORTANT: +# If we want a namelist variable to be removed from the namelist file, +# in the "settings" variable below, we need to set its value to the +# string "null". This is equivalent to setting its value to +# !!python/none +# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the +# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. +# +# It turns out that setting the variable to an empty string also works +# to remove it from the namelist! Which is better to use?? +# settings=" 'config': { 'fix_dir_input_grid': ${FIXgsm}, diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh new file mode 100644 index 0000000000..db2edaed2f --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GSD_SAR" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 944832518a..8b1df05aa9 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -30,6 +30,7 @@ grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE diff --git a/tests/get_expts_status.sh b/tests/get_expts_status.sh new file mode 100755 index 0000000000..3b65422e3e --- /dev/null +++ b/tests/get_expts_status.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script checks the workflow status of all forecast experiments +# with directories located under a given base directory (expts_subdir). +# It must be supplied exactly one argument -- the experiments base +# directory. It assumes that all subdirectories under this base directory +# are experiment directories and checks each one's workflow status. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Do not allow uninitialized variables. +# +#----------------------------------------------------------------------- +# +set -u +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# The current script should be located in the "tests" subdirectory of the +# workflow's top-level directory, which we denote by homerrfs. Thus, +# homerrfs is the directory one level above the directory in which the +# current script is located. Set homerrfs accordingly. +# +#----------------------------------------------------------------------- +# +homerrfs=${scrfunc_dir%/*} +# +#----------------------------------------------------------------------- +# +# Set directories. +# +#----------------------------------------------------------------------- +# +ushdir="$homerrfs/ush" +# +#----------------------------------------------------------------------- +# +# Source bash utility functions. +# +#----------------------------------------------------------------------- +# +. $ushdir/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Exactly one argument must be specified that consists of the full path +# to the experiments base directory (i.e. the directory containing the +# experiment subdirectories). Ensure that the number of arguments is +# one. +# +#----------------------------------------------------------------------- +# +num_args="$#" +if [ "${num_args}" -eq 1 ]; then + expts_basedir="$1" +else + print_err_msg_exit " +The number of arguments to this script must be exacty one, and that +argument must specify the experiments base directory, i.e. the directory +containing the experiment subdirectories. The acutal number of arguments +is: + num_args = ${num_args}" +fi +# +#----------------------------------------------------------------------- +# +# Check that the specified experiments base directory exists and is +# actually a directory. If not, print out an error message and exit. +# If so, print out an informational message. +# +#----------------------------------------------------------------------- +# +if [ ! -d "${expts_basedir}" ]; then + print_err_msg_exit " +The experiments base directory (expts_basedir) does not exit or is not +actually a directory: + expts_basedir = \"${expts_basedir}\"" +else + print_info_msg " +Checking the workflow status of all forecast experiments in the following +specified experiments base directory: + expts_basedir = \"${expts_basedir}\"" +fi +# +#----------------------------------------------------------------------- +# +# Create an array containing the names of the subdirectories in the +# experiment base directory. +# +#----------------------------------------------------------------------- +# +cd "${expts_basedir}" +# Get a list of all subdirectories (but not files) in the experiment base +# directory. Note that the ls command below will return a string containing +# the subdirectory names, with each name followed by a backslash and a +# newline. +expts_list=$( \ls -1 -d */ ) +# Remove all backslashes from the ends of the subdirectory names. +expts_list=$( printf "${expts_list}" "%s" | sed -r 's|/||g' ) +# Create an array out of the string containing the newline-separated list +# of experiment subdirectories. +expts_list=( ${expts_list} ) +# +#----------------------------------------------------------------------- +# +# Get the number of experiments for which to check the workflow status +# and print out an informational message. +# +#----------------------------------------------------------------------- +# +num_expts="${#expts_list[@]}" +expts_list_str=$( printf " \'%s\'\n" "${expts_list[@]}" ) +print_info_msg " +The number of experiments found is: + num_expts = ${num_expts} +The list of experiments whose workflow status will be checked is: +${expts_list_str} +" +# +#----------------------------------------------------------------------- +# +# Set the name and full path of the file in which the status report will +# be saved. If such a file already exists, rename it. +# +#----------------------------------------------------------------------- +# +yyyymmddhhmn=$( date +%Y%m%d%H%M ) +expts_status_fn="expts_status_${yyyymmddhhmn}.txt" +expts_status_fp="${expts_basedir}/${expts_status_fn}" + +# Note that the check_for_preexist_dir_file function assumes that there +# is a variable named "VERBOSE" in the environment. Set that before +# calling the function. +VERBOSE="TRUE" +check_for_preexist_dir_file "${expts_status_fp}" "rename" +# +#----------------------------------------------------------------------- +# +# Loop through the elements of the array expts_list. For each element +# (i.e. for each experiment), change location to the experiment directory +# and call the script launch_FV3LAM_wflow.sh to update the log file +# log.launch_FV3LAM_wflow. Then take the last num_tail_lines of this +# log file (along with an appropriate message) and add it to the status +# report file. +# +#----------------------------------------------------------------------- +# +num_tail_lines="40" + +for (( i=0; i<=$((num_expts-1)); i++ )); do + + expt_subdir="${expts_list[$i]}" + msg=" +====================================== +Checking workflow status of experiment: \"${expt_subdir}\"" + print_info_msg "$msg" +# +# Change location to the experiment subdirectory, call the workflow launch +# script to update the launch log file, and capture the output from that +# call. +# + cd_vrfy "${expt_subdir}" + launch_msg=$( launch_FV3LAM_wflow.sh 2>&1 ) + log_tail=$( tail -n ${num_tail_lines} log.launch_FV3LAM_wflow ) +# +# Record the tail from the log file into the status report file. +# + print_info_msg "$msg" >> "${expts_status_fp}" + print_info_msg "${log_tail}" >> "${expts_status_fp}" +# +# Print the workflow status to the screen. + wflow_status=$( printf "${log_tail}" | grep "Workflow status:" ) +# wflow_status="${wflow_status## }" # Not sure why this doesn't work to strip leading spaces. + wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) + msg="\ +${wflow_status} +====================================== +" + print_info_msg "$msg" +# +# Change location back to the experiments base directory. +# + cd_vrfy "${expts_basedir}" + +done + +print_info_msg "\ +DONE." diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 7b87c8fc1e..9e3143aed6 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -1,13 +1,15 @@ # #----------------------------------------------------------------------- # -# This file defins a function that first checks whether the Thompson -# microphysics parameterization is being called by the selected physics -# suite. If not, it does nothing else. If so, it modifies the workflow -# arrays FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING -# to ensure that fixed files needed by this parameterization are copied -# to the FIXam directory and that appropriate symlinks to these files -# are created in the run directories. +# This file defines a function that first checks whether the Thompson +# microphysics parameterization is being called by the selected physics +# suite. If not, it sets the output variable specified by +# output_varname_thompson_mp_used to "FALSE" and exits. If so, it sets +# this variable to "TRUE" and modifies the workflow arrays +# FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING +# to ensure that fixed files needed by the Thompson microphysics +# parameterization are copied to the FIXam directory and that appropriate +# symlinks to these files are created in the run directories. # #----------------------------------------------------------------------- # @@ -52,6 +54,8 @@ function set_thompson_mp_fix_files() { # local valid_args=( \ "ccpp_phys_suite_fp" \ + "thompson_mp_climo_fn" \ + "output_varname_thompson_mp_used" \ ) process_args valid_args "$@" # @@ -74,7 +78,7 @@ function set_thompson_mp_fix_files() { local thompson_mp_name \ regex_search \ thompson_mp_name_or_null \ - thompson_mp_is_used \ + thompson_mp_used \ thompson_mp_fix_files \ num_files \ mapping \ @@ -92,9 +96,9 @@ function set_thompson_mp_fix_files() { thompson_mp_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) if [ "${thompson_mp_name_or_null}" = "${thompson_mp_name}" ]; then - thompson_mp_is_used="TRUE" + thompson_mp_used="TRUE" elif [ -z "${thompson_mp_name_or_null}" ]; then - thompson_mp_is_used="FALSE" + thompson_mp_used="FALSE" else print_err_msg_exit "\ Unexpected value returned for thompson_mp_name_or_null: @@ -105,29 +109,32 @@ string." # #----------------------------------------------------------------------- # -# If the Thompson microphysics parameterization is being used, then -# append the names of the fixed files needed by this parameterization to -# the workflow array FIXgsm_FILES_TO_COPY_TO_FIXam, and append to the -# workflow array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING the mappings -# between these files and the names of the corresponding symlinks that -# need to be created in the run directories. +# If the Thompson microphysics parameterization is being used, then... +# +#----------------------------------------------------------------------- +# + if [ "${thompson_mp_used}" = "TRUE" ]; then +# +#----------------------------------------------------------------------- +# +# Append the names of the fixed files needed by the Thompson microphysics +# parameterization to the workflow array FIXgsm_FILES_TO_COPY_TO_FIXam, +# and append to the workflow array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING +# the mappings between these files and the names of the corresponding +# symlinks that need to be created in the run directories. # #----------------------------------------------------------------------- # - if [ "${thompson_mp_is_used}" = "TRUE" ]; then - thompson_mp_fix_files=( \ "CCN_ACTIVATE.BIN" \ "freezeH2O.dat" \ "qr_acr_qg.dat" \ "qr_acr_qs.dat" \ ) - + if [ "${EXTRN_MDL_NAME_ICS}" != "HRRR" -a "${EXTRN_MDL_NAME_ICS}" != "RAP" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" != "HRRR" -a "${EXTRN_MDL_NAME_LBCS}" != "RAP" ]; then - - thompson_mp_fix_files+=( "Thompson_MP_MONTHLY_CLIMO.nc" ) - + thompson_mp_fix_files+=( "${thompson_mp_climo_fn}" ) fi FIXgsm_FILES_TO_COPY_TO_FIXam+=( "${thompson_mp_fix_files[@]}" ) @@ -165,6 +172,14 @@ values of these parameters are as follows: # #----------------------------------------------------------------------- # +# Set output variables. +# +#----------------------------------------------------------------------- +# + eval ${output_varname_thompson_mp_used}="${thompson_mp_used}" +# +#----------------------------------------------------------------------- +# # Restore the shell options saved at the beginning of this script/function. # #----------------------------------------------------------------------- diff --git a/ush/setup.sh b/ush/setup.sh index 4a6d0b3335..8f23bbefea 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1344,21 +1344,19 @@ LOAD_MODULES_RUN_TASK_FP="$USHDIR/load_modules_run_task.sh" # processing, as follows: # # GRID_DIR: -# Directory in which the grid files will be placed (if RUN_TASK_MAKE_- -# GRID is set to "TRUE") or searched for (if RUN_TASK_MAKE_GRID is set -# to "FALSE"). +# Directory in which the grid files will be placed (if RUN_TASK_MAKE_GRID +# is set to "TRUE") or searched for (if RUN_TASK_MAKE_GRID is set to +# "FALSE"). # # OROG_DIR: -# Directory in which the orography files will be placed (if RUN_TASK_- -# MAKE_OROG is set to "TRUE") or searched for (if RUN_TASK_MAKE_OROG is -# set to "FALSE"). +# Directory in which the orography files will be placed (if RUN_TASK_MAKE_OROG +# is set to "TRUE") or searched for (if RUN_TASK_MAKE_OROG is set to +# "FALSE"). # # SFC_CLIMO_DIR: # Directory in which the surface climatology files will be placed (if -# RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE") or searched for (if RUN_- -# TASK_MAKE_SFC_CLIMO is set to "FALSE"). -# -# First, consider NCO mode. +# RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE") or searched for (if +# RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE"). # #---------------------------------------------------------------------- # @@ -1669,9 +1667,9 @@ NH4=4 # #----------------------------------------------------------------------- # -# Set parameters according to the type of horizontal grid generation me- -# thod specified. First consider GFDL's global-parent-grid based me- -# thod. +# Set parameters according to the type of horizontal grid generation +# method specified. First consider GFDL's global-parent-grid based +# method. # #----------------------------------------------------------------------- # @@ -1956,8 +1954,21 @@ NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - - +# +#----------------------------------------------------------------------- +# +# Set the name of the file containing aerosol climatology data that, if +# necessary, can be used to generate approximate versions of the aerosol +# fields needed by Thompson microphysics. This file will be used to +# generate such approximate aerosol fields in the ICs and LBCs if Thompson +# MP is included in the physics suite and if the exteranl model for ICs +# or LBCs does not already provide these fields. Also, set the full path +# to this file. +# +#----------------------------------------------------------------------- +# +THOMPSON_MP_CLIMO_FN="Thompson_MP_MONTHLY_CLIMO.nc" +THOMPSON_MP_CLIMO_FP="$FIXam/${THOMPSON_MP_CLIMO_FN}" # #----------------------------------------------------------------------- # @@ -1965,16 +1976,16 @@ NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) # is being called by the physics suite, modifies certain workflow arrays # to ensure that fixed files needed by this parameterization are copied # to the FIXam directory and appropriate symlinks to them are created in -# the run directories. +# the run directories. This function also sets the workflow variable +# THOMPSON_MP_USED that indicates whether Thompson MP is called by the +# physics suite. # #----------------------------------------------------------------------- # set_thompson_mp_fix_files \ - ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" - - - - + ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ + thompson_mp_climo_fn="${THOMPSON_MP_CLIMO_FN}" \ + output_varname_thompson_mp_used="THOMPSON_MP_USED" # #----------------------------------------------------------------------- # @@ -2357,6 +2368,18 @@ NEMS_CONFIG_FP="${NEMS_CONFIG_FP}" FV3_EXEC_FP="${FV3_EXEC_FP}" LOAD_MODULES_RUN_TASK_FP="${LOAD_MODULES_RUN_TASK_FP}" + +THOMPSON_MP_CLIMO_FN="${THOMPSON_MP_CLIMO_FN}" +THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" +# +#----------------------------------------------------------------------- +# +# Parameters that indicate whether or not various parameterizations are +# included in and called by the phsics suite. +# +#----------------------------------------------------------------------- +# +THOMPSON_MP_USED="${THOMPSON_MP_USED}" # #----------------------------------------------------------------------- # From eda1ac58a781d5a78fe826edfedb971665c9709d Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 27 Jan 2021 01:28:22 -0700 Subject: [PATCH 014/203] Improve the way LSM-related parameters are set. (#416) ## DESCRIPTION OF CHANGES: ### Main Changes: * Add the new function check_ruc_lsm.sh that reads in the SDF to check whether it uses the RUC LSM. If so, it sets the local variable sdf_uses_ruc_lsm to "TRUE", otherwise to "FALSE". It then sets the environment variable whose name is specified by the input argument output_varname_sdf_uses_ruc_lsm to the value of sdf_uses_ruc_lsm. * In setup.sh, introduce new workflow variable SDF_USES_RUC_LSM by passing its name as an argument to the new function check_ruc_lsm.sh. This gets set to "TRUE" if the physics suite uses the RUC LSM and "FALSE" otherwise. * In exregional_make_ics.sh, set the chgres_cube namelist variable nsoill_out (which is the number of soil levels to have in the NetCDF file that chgres_cube generates) by uisng an if-statement that checks the new workflow variable SDF_USES_RUC_LSM as well as the name of the external model used for ICs. This if-statemnt replaces the settings of nsoill_out in the external-model-dependent case-statement and, depending on model, the physics-suite-dependent if-statements within that case statement. This simplifies the script because we no longer need to remember which suites use the RUC LSM and which don't. * In generate_FV3LAM_wflow.sh, set the number of ice levels in the FV3 input file according to whether the RUC_LSM is used in the physics suite (i.e. by checking the new variable SDF_USES_RUC_LSM). ### Improvements: * Change the name of the workflow variable THOMPSON_MP_USED to SDF_USES_THOMPSON_MP to be consistent with the new varable SDF_USES_RUC_LSM. * In set_thompson_mp_fix_files.sh, change the local variable thompson_mp_used to sdf_uses_thompson_mp (to be consistent with the change in the corresponding workflow variable SDF_USES_THOMPSON_MP). ## TESTS CONDUCTED: Ran the following WE2E tests on Hera using the HEAD (hash #ea8a7aa) of the authoritative ufs-community/ufs-weather-model repo: * grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * nco_CONUS_25km_GFDLgrid * nco_RRFS_CONUS_25km_HRRR_RAP * suite_FV3_GSD_v0 All tests were successful. Note that the three tests whose names end with "_2020020800" are not in the workflow, but they are generated by taking the respective WE2E tests in the workflow with names without the trailing "_2020020800" and simply changing DATE_FIRST_CYCL and DATE_LAST_CYCL from 20200801 to 20200208 and LBC_SPEC_INTVL_HRS from 3 to 1. The change in date was to try out a winter case, which was the situation under which failures were common (the setting of kice to 9 for suites that use the RUC LSM seems to have fixed this winter case problem). --- scripts/exregional_make_ics.sh | 116 ++++++++++++++++-------------- scripts/exregional_make_lbcs.sh | 33 ++++++--- ush/check_ruc_lsm.sh | 120 +++++++++++++++++++++++++++++++ ush/generate_FV3LAM_wflow.sh | 65 ++++++++++------- ush/set_thompson_mp_fix_files.sh | 18 ++--- ush/setup.sh | 23 ++++-- 6 files changed, 271 insertions(+), 104 deletions(-) create mode 100644 ush/check_ruc_lsm.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index c81efddf2c..7b8c574fbd 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -302,22 +302,73 @@ lai_from_climo="" tg3_from_soil="" convert_nst="" # -# If the physics suite uses Thompson microphysics and if the external -# model for ICs is one that does not provide the aerosol fields needed -# by Thompson microphysics (currently only the HRRR and RAP provide -# aerosol data), set the variable thomp_mp_climo_file in the chgres_cube -# namelist to the full path of the file containing aerosol climatology -# data. In this case, this file will be used to generate approximate +#----------------------------------------------------------------------- +# +# If the external model is not one that uses the RUC land surface model +# (LSM) -- which currently includes all valid external models except the +# HRRR and the RAP -- then we set the number of soil levels to include +# in the output NetCDF file that chgres_cube generates (nsoill_out; this +# is a variable in the namelist that chgres_cube reads in) to 4. This +# is because FV3 can handle this regardless of the LSM that it is using +# (which is specified in the suite definition file, or SDF), as follows. +# If the SDF does not use the RUC LSM (i.e. it uses the Noah or Noah MP +# LSM), then it will expect to see 4 soil layers; and if the SDF uses +# the RUC LSM, then the RUC LSM itself has the capability to regrid from +# 4 soil layers to the 9 layers that it uses. +# +# On the other hand, if the external model is one that uses the RUC LSM +# (currently meaning that it is either the HRRR or the RAP), then what +# we set nsoill_out to depends on whether the RUC or the Noah/Noah MP +# LSM is used in the SDF. If the SDF uses RUC, then both the external +# model and FV3 use RUC (which expects 9 soil levels), so we simply set +# nsoill_out to 9. In this case, chgres_cube does not need to do any +# regridding of soil levels (because the number of levels in is the same +# as the number out). If the SDF uses the Noah or Noah MP LSM, then the +# output from chgres_cube must contain 4 soil levels because that is what +# these LSMs expect, and the code in FV3 does not have the capability to +# regrid from the 9 levels in the external model to the 4 levels expected +# by Noah/Noah MP. In this case, chgres_cube does the regridding from +# 9 to 4 levels. +# +# In summary, we can set nsoill_out to 4 unless the external model is +# the HRRR or RAP AND the forecast model is using the RUC LSM. +# +#----------------------------------------------------------------------- +# +nsoill_out="4" +if [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" -o \ + "${EXTRN_MDL_NAME_ICS}" = "RAP" ] && \ + [ "${SDF_USES_RUC_LSM}" = "TRUE" ]; then + nsoill_out="9" +fi +# +#----------------------------------------------------------------------- +# +# If the external model for ICs is one that does not provide the aerosol +# fields needed by Thompson microphysics (currently only the HRRR and +# RAP provide aerosol data) and if the physics suite uses Thompson +# microphysics, set the variable thomp_mp_climo_file in the chgres_cube +# namelist to the full path of the file containing aerosol climatology +# data. In this case, this file will be used to generate approximate # aerosol fields in the ICs that Thompson MP can use. Otherwise, set # thomp_mp_climo_file to a null string. # +#----------------------------------------------------------------------- +# thomp_mp_climo_file="" -if [ "${THOMPSON_MP_USED}" = "TRUE" ] && \ - [ "${EXTRN_MDL_NAME_ICS}" != "HRRR" -a \ - "${EXTRN_MDL_NAME_ICS}" != "RAP" ]; then +if [ "${EXTRN_MDL_NAME_ICS}" != "HRRR" -a \ + "${EXTRN_MDL_NAME_ICS}" != "RAP" ] && \ + [ "${SDF_USES_THOMPSON_MP}" = "TRUE" ]; then thomp_mp_climo_file="${THOMPSON_MP_CLIMO_FP}" fi - +# +#----------------------------------------------------------------------- +# +# Set other chgres_cube namelist variables depending on the external +# model used. +# +#----------------------------------------------------------------------- +# case "${EXTRN_MDL_NAME_ICS}" in "GSMGFS") @@ -328,7 +379,6 @@ case "${EXTRN_MDL_NAME_ICS}" in convert_nst=False tracers_input="[\"spfh\",\"clwmr\",\"o3mr\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\"]" - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True vgfrc_from_climo=True @@ -352,7 +402,6 @@ case "${EXTRN_MDL_NAME_ICS}" in input_type="grib2" convert_nst=False fi - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True vgfrc_from_climo=True @@ -366,27 +415,6 @@ case "${EXTRN_MDL_NAME_ICS}" in fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" # -# Set soil levels based on LSM in CCPP SDF (RUC-LSM or Noah/Noah MP). -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - nsoill_out="4" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - nsoill_out="9" - else - print_err_msg_exit "\ -The variable \"nsoill_out\" has not yet been specified for this external -IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi -# # Path to the HRRRX geogrid file. # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_HRRRX" @@ -406,27 +434,6 @@ IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" # -# Set soil levels based on CCPP SDF. -# - if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_2017_gfdlmp_regional" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_CPT_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15p2" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v16beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1alpha" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then - nsoill_out="4" - elif [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - nsoill_out="9" - else - print_err_msg_exit "\ -The variable \"nsoill_out\" has not yet been specified for this external -IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" - fi -# # Path to the RAPX geogrid file. # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_RAPX" @@ -443,7 +450,6 @@ IC model (EXTRN_MDL_NAME_ICS) and physics suite (CCPP_PHYS_SUITE) combination: external_model="NAM" fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" - nsoill_out="4" #If the CCPP suites uses RUC-LSM, the scheme will interpolate from 4 to 9 soil levels. vgtyp_from_climo=True sotyp_from_climo=True vgfrc_from_climo=True diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 5205b74e10..d2f59eb1f4 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -274,22 +274,33 @@ input_type="" tracers_input="\"\"" tracers="\"\"" # -# If the physics suite uses Thompson microphysics and if the external -# model for LBCs is one that does not provide the aerosol fields needed -# by Thompson microphysics (currently only the HRRR and RAP provide -# aerosol data), set the variable thomp_mp_climo_file in the chgres_cube -# namelist to the full path of the file containing aerosol climatology -# data. In this case, this file will be used to generate approximate -# aerosol fields in the LBCs that Thompson MP can use. Otherwise, set +#----------------------------------------------------------------------- +# +# If the external model for LBCs is one that does not provide the aerosol +# fields needed by Thompson microphysics (currently only the HRRR and +# RAP provide aerosol data) and if the physics suite uses Thompson +# microphysics, set the variable thomp_mp_climo_file in the chgres_cube +# namelist to the full path of the file containing aerosol climatology +# data. In this case, this file will be used to generate approximate +# aerosol fields in the LBCs that Thompson MP can use. Otherwise, set # thomp_mp_climo_file to a null string. # +#----------------------------------------------------------------------- +# thomp_mp_climo_file="" -if [ "${THOMPSON_MP_USED}" = "TRUE" ] && \ - [ "${EXTRN_MDL_NAME_LBCS}" != "HRRR" -a \ - "${EXTRN_MDL_NAME_LBCS}" != "RAP" ]; then +if [ "${EXTRN_MDL_NAME_LBCS}" != "HRRR" -a \ + "${EXTRN_MDL_NAME_LBCS}" != "RAP" ] && \ + [ "${SDF_USES_THOMPSON_MP}" = "TRUE" ]; then thomp_mp_climo_file="${THOMPSON_MP_CLIMO_FP}" fi - +# +#----------------------------------------------------------------------- +# +# Set other chgres_cube namelist variables depending on the external +# model used. +# +#----------------------------------------------------------------------- +# case "${EXTRN_MDL_NAME_LBCS}" in "GSMGFS") diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh new file mode 100644 index 0000000000..fe22d08abc --- /dev/null +++ b/ush/check_ruc_lsm.sh @@ -0,0 +1,120 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that checks whether the RUC land surface +# model (LSM) parameterization is being called by the selected physics +# suite. If so, it sets the variable ruc_lsm used to "TRUE". If not, +# it sets this variable to "FALSE". It then "returns" this variable, +# i.e. it sets the environment variable whose name is specified by the +# input argument output_varname_sdf_uses_ruc_lsm to whatever sdf_uses_ruc_lsm +# is set to. +# +#----------------------------------------------------------------------- +# +function check_ruc_lsm() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "ccpp_phys_suite_fp" \ + "output_varname_sdf_uses_ruc_lsm" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local ruc_lsm_name \ + regex_search \ + ruc_lsm_name_or_null \ + sdf_uses_ruc_lsm +# +#----------------------------------------------------------------------- +# +# Check the suite definition file to see whether the Thompson microphysics +# parameterization is being used. +# +#----------------------------------------------------------------------- +# + ruc_lsm_name="lsm_ruc" + regex_search="^[ ]*(${ruc_lsm_name})<\/scheme>[ ]*$" + ruc_lsm_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) + + if [ "${ruc_lsm_name_or_null}" = "${ruc_lsm_name}" ]; then + sdf_uses_ruc_lsm="TRUE" + elif [ -z "${ruc_lsm_name_or_null}" ]; then + sdf_uses_ruc_lsm="FALSE" + else + print_err_msg_exit "\ +Unexpected value returned for ruc_lsm_name_or_null: + ruc_lsm_name_or_null = \"${ruc_lsm_name_or_null}\" +This variable should be set to either \"${ruc_lsm_name}\" or an empty +string." + fi +# +#----------------------------------------------------------------------- +# +# Set output variables. +# +#----------------------------------------------------------------------- +# + eval ${output_varname_sdf_uses_ruc_lsm}="${sdf_uses_ruc_lsm}" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 6701c25c20..03cc137dc0 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -508,30 +508,35 @@ Setting parameters in FV3 namelist file (FV3_NML_FP): npx=$((NX+1)) npy=$((NY+1)) # -# For the physics suites that use RUC-LSM, set the parameter -# lsoil according to the external models used to obtain ICs and LBCs. -# -if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - - if [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - lsoil=4 - elif [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ]; then - lsoil=9 - else - print_err_msg_exit "\ -The value to set the variable lsoil to in the FV3 namelist file (FV3_NML_FP) -has not been specified for the following combination of physics suite and -external model for ICs: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" -Please change one or more of these parameters or provide a value for lsoil -(and change workflow generation script(s) accordingly) and rerun." - fi - +# For the physics suites that use RUC LSM, set the parameter kice to 9, +# Otherwise, leave it unspecified (which means it gets set to the default +# value in the forecast model). +# +# NOTE: +# May want to remove kice from FV3.input.yml (and maybe input.nml.FV3). +# +kice="" +if [ "${SDF_USES_RUC_LSM}" = "TRUE" ]; then + kice="9" +fi +# +# Set lsoil, which is the number of input soil levels provided in the +# chgres_cube output NetCDF file. This is the same as the parameter +# nsoill_out in the namelist file for chgres_cube. [On the other hand, +# the parameter lsoil_lsm (not set here but set in input.nml.FV3 and/or +# FV3.input.yml) is the number of soil levels that the LSM scheme in the +# forecast model will run with.] Here, we use the same approach to set +# lsoil as the one used to set nsoill_out in exregional_make_ics.sh. +# See that script for details. +# +# NOTE: +# May want to remove lsoil from FV3.input.yml (and maybe input.nml.FV3). +# +lsoil="4" +if [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" -o \ + "${EXTRN_MDL_NAME_ICS}" = "RAP" ] && \ + [ "${SDF_USES_RUC_LSM}" = "TRUE" ]; then + lsoil="9" fi # # Create a multiline variable that consists of a yaml-compliant string @@ -540,6 +545,17 @@ fi # passed to a python script that will in turn set the values of these # variables in the namelist file. # +# IMPORTANT: +# If we want a namelist variable to be removed from the namelist file, +# in the "settings" variable below, we need to set its value to the +# string "null". This is equivalent to setting its value to +# !!python/none +# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the +# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. +# +# It turns out that setting the variable to an empty string also works +# to remove it from the namelist! Which is better to use?? +# settings="\ 'atmos_model_nml': { 'blocksize': $BLOCKSIZE, @@ -564,6 +580,7 @@ settings="\ 'bc_update_interval': ${LBC_SPEC_INTVL_HRS}, } 'gfs_physics_nml': { + 'kice': ${kice:-null}, 'lsoil': ${lsoil:-null}, 'do_shum': ${DO_SHUM}, 'do_sppt': ${DO_SPPT}, diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 9e3143aed6..c5f1d67e5a 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -3,9 +3,9 @@ # # This file defines a function that first checks whether the Thompson # microphysics parameterization is being called by the selected physics -# suite. If not, it sets the output variable specified by -# output_varname_thompson_mp_used to "FALSE" and exits. If so, it sets -# this variable to "TRUE" and modifies the workflow arrays +# suite. If not, it sets the output variable whose name is specified by +# output_varname_sdf_uses_thompson_mp to "FALSE" and exits. If so, it +# sets this variable to "TRUE" and modifies the workflow arrays # FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING # to ensure that fixed files needed by the Thompson microphysics # parameterization are copied to the FIXam directory and that appropriate @@ -55,7 +55,7 @@ function set_thompson_mp_fix_files() { local valid_args=( \ "ccpp_phys_suite_fp" \ "thompson_mp_climo_fn" \ - "output_varname_thompson_mp_used" \ + "output_varname_sdf_uses_thompson_mp" \ ) process_args valid_args "$@" # @@ -78,7 +78,7 @@ function set_thompson_mp_fix_files() { local thompson_mp_name \ regex_search \ thompson_mp_name_or_null \ - thompson_mp_used \ + sdf_uses_thompson_mp \ thompson_mp_fix_files \ num_files \ mapping \ @@ -96,9 +96,9 @@ function set_thompson_mp_fix_files() { thompson_mp_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) if [ "${thompson_mp_name_or_null}" = "${thompson_mp_name}" ]; then - thompson_mp_used="TRUE" + sdf_uses_thompson_mp="TRUE" elif [ -z "${thompson_mp_name_or_null}" ]; then - thompson_mp_used="FALSE" + sdf_uses_thompson_mp="FALSE" else print_err_msg_exit "\ Unexpected value returned for thompson_mp_name_or_null: @@ -113,7 +113,7 @@ string." # #----------------------------------------------------------------------- # - if [ "${thompson_mp_used}" = "TRUE" ]; then + if [ "${sdf_uses_thompson_mp}" = "TRUE" ]; then # #----------------------------------------------------------------------- # @@ -176,7 +176,7 @@ values of these parameters are as follows: # #----------------------------------------------------------------------- # - eval ${output_varname_thompson_mp_used}="${thompson_mp_used}" + eval ${output_varname_sdf_uses_thompson_mp}="${sdf_uses_thompson_mp}" # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index 8f23bbefea..aa72485617 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -63,6 +63,7 @@ cd_vrfy ${scrfunc_dir} . ./link_fix.sh . ./set_ozone_param.sh . ./set_thompson_mp_fix_files.sh +. ./check_ruc_lsm.sh # #----------------------------------------------------------------------- # @@ -1953,7 +1954,18 @@ fi NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - +# +#----------------------------------------------------------------------- +# +# Call the function that checks whether the RUC land surface model (LSM) +# is being called by the physics suite and sets the workflow variable +# SDF_USES_RUC_LSM to "TRUE" or "FALSE" accordingly. +# +#----------------------------------------------------------------------- +# +check_ruc_lsm \ + ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ + output_varname_sdf_uses_ruc_lsm="SDF_USES_RUC_LSM" # #----------------------------------------------------------------------- # @@ -1977,15 +1989,15 @@ THOMPSON_MP_CLIMO_FP="$FIXam/${THOMPSON_MP_CLIMO_FN}" # to ensure that fixed files needed by this parameterization are copied # to the FIXam directory and appropriate symlinks to them are created in # the run directories. This function also sets the workflow variable -# THOMPSON_MP_USED that indicates whether Thompson MP is called by the -# physics suite. +# SDF_USES_THOMPSON_MP that indicates whether Thompson MP is called by +# the physics suite. # #----------------------------------------------------------------------- # set_thompson_mp_fix_files \ ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ thompson_mp_climo_fn="${THOMPSON_MP_CLIMO_FN}" \ - output_varname_thompson_mp_used="THOMPSON_MP_USED" + output_varname_sdf_uses_thompson_mp="SDF_USES_THOMPSON_MP" # #----------------------------------------------------------------------- # @@ -2379,7 +2391,8 @@ THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" # #----------------------------------------------------------------------- # -THOMPSON_MP_USED="${THOMPSON_MP_USED}" +SDF_USES_RUC_LSM="${SDF_USES_RUC_LSM}" +SDF_USES_THOMPSON_MP="${SDF_USES_THOMPSON_MP}" # #----------------------------------------------------------------------- # From 5bb994d10c7dcf869bf3aa8c21dcbf8edecaaa9f Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 27 Jan 2021 05:21:40 -0700 Subject: [PATCH 015/203] Add the FV3_HRRR suite to the workflow (#382) ## DESCRIPTION OF CHANGES: ### Main changes: * Add the FV3_HRRR suite to the workflow. This consists of: * Adding "FV3_HRRR" to the list of valid values of CCPP_PHYS_SUITE in valid_param_vals.sh. * Adding the diag_table.FV3_HRRR and field_table.FV3_HRRR files to the ush/templates directory. * Adding the FV3_HRRR suite to the case-statements in exregional_make_ics.sh and exregional_make_lbcs.sh. * Adding a stanza to FV3.input.yml to customize the FV3 namelist file for the FV3_HRRR suite. * Add the following two WE2E tests for the FV3_HRRR suite on the RRFS_CONUS_25km grid. The first is initialized for the 2020 derecho case (2020081000) and updates the LBCs every hour while the second is initialized on 2019052000 and updates the LBCs every 6 hours: * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR * suite_FV3_HRRR * Add an additional three WE2E tests for the FV3_HRRR suite on the 25km, 13km, and 3km RRFS_CONUS grids. All three are initialized for the 2020 derecho case (2020081000). They are named as follows: * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR * Modify the following three WE2E tests that use the FV3_GSD_SAR suite so that they are initialized for the 2020 derecho case (2020081000) instead of 2020080100 (so that they are consistent with the 3 new WE2E tests added above): * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * Enable the workflow to find and copy to the experiment directory the pregenerated (for a given grid) orography statistics files `${CRES}_oro_data_ls.tile7.halo0.nc` and `${CRES}_oro_data_ss.tile7.halo0.nc` needed by the gravity wave drag parameterization in FV3_HRRR. (Note that this is a temporary fix. Eventually, a program will be made available that the make_orog task can call to generate these files as part of the workflow for any grid.) Modifications for this purpose consist of: * Adding to config_defaults.sh the new workflow variable GWD_HRRRsuite_BASEDIR that specifies the base directory in which the aformentioned "ls" and "ss" orography statistics files are located for the predefined grid being used. This workflow variable will be removed once a program is available to generate the "ls" and "ss" files on-the-fly for any grid. * Adding code to setup.sh that sets the new derived/secondary workflow variable GWD_HRRRsuite_DIR that specifies the directory in which the "ls" and "ss" files are located (as opposed to just the base directory, which is what GWD_HRRRsuite_BASEDIR represents). This is set using GWD_HRRRsuite_BASEDIR and PREDEF_GRID_NAME. Like GWD_HRRRsuite_BASEDIR, this variable will be removed once a program is available to generate the "ls" and "ss" files on-the-fly for any grid. * Adding code to link_fix.sh that creates symlinks in the fv3_lam directory that point to the "ls" and "ss" files in OROG_DIR. * Adding code to exregional_make_orog.sh that copies the "ls" and "ss" files from the directory specified by GWD_HRRRsuite_DIR to the orography directory (OROG_DIR). This code will evenutally be replaced by a call to a program that will generate these files and place them in OROG_DIR (instead of copying them). * Adding code to exregional_run_fcst.sh that creates links in the run directory to the"ls" and "ss" orography files that the FV3 model looks for. * Adding code to run_experiments.sh that sets GWD_HRRRsuite_BASEDIR in the workflow configuration file (config.sh) to the base directory in which the pregenerated "ls" and "ss" orography files are located. Note that this setting is machine-dependent. ### Improvements: * Edit error messages in exregional_make_ics.sh and exregional_make_lbcs.sh to make them clearer. * Correct code indentation and remove blank lines. * Alphabetize FV3_GSD_SAR stanza in FV3.input.yml. * In tests/run_experiments.sh, change the location FIXLAM_NCO_BASEDIR on Jet to be a more "official" directory (instead of a personal directory). * Add the WE2E test grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR that uses the HRRRX external model for both the ICs and the LBCs (and uses the FV3_GSD_SAR suite). * Clean up comments. ## TESTS CONDUCTED: Ran the following WE2E tests on Hera: * grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR_2020020800 * grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR_2020020800 * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR_2020020800 * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR_2020020800 * grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * nco_CONUS_25km_GFDLgrid * nco_RRFS_CONUS_25km_HRRR_RAP * suite_FV3_GSD_v0 * suite_FV3_HRRR All tests were successful. Note that the six tests whose names end with "_2020020800" are not in the workflow. They are generated by taking the respective WE2E tests in the workflow with names without the trailing "_2020020800" and simply changing DATE_FIRST_CYCL and DATE_LAST_CYCL from 20200801 to 20200208 and LBC_SPEC_INTVL_HRS from 3 to 1. The change in date was to try out a winter case. ## CONTRIBUTORS (optional): @mdtoy --- scripts/exregional_make_ics.sh | 3 +- scripts/exregional_make_lbcs.sh | 3 +- scripts/exregional_make_orog.sh | 16 + scripts/exregional_run_fcst.sh | 27 ++ ...US_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 4 +- ...CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 17 + ...NUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh | 19 + ...S_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh | 17 + ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 4 +- ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 17 + ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 4 +- ..._CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 17 + tests/baselines_list.txt | 5 + tests/run_experiments.sh | 37 +- ush/config_defaults.sh | 21 + ush/generate_FV3LAM_wflow.sh | 1 + ush/link_fix.sh | 6 + ush/setup.sh | 34 ++ ush/templates/FV3.input.yml | 29 +- ush/templates/diag_table.FV3_HRRR | 361 ++++++++++++++++++ ush/templates/field_table.FV3_HRRR | 65 ++++ ush/valid_param_vals.sh | 1 + 22 files changed, 698 insertions(+), 10 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh create mode 100644 ush/templates/diag_table.FV3_HRRR create mode 100644 ush/templates/field_table.FV3_HRRR diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 7b8c574fbd..f335fa1635 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -160,7 +160,8 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GSD_v0" | \ "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ - "FV3_RRFS_v1beta" ) + "FV3_RRFS_v1beta" | \ + "FV3_HRRR" ) if [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ]; then varmap_file="GSDphys_var_map.txt" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index d2f59eb1f4..6a9fc7365e 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -160,7 +160,8 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GSD_v0" | \ "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ - "FV3_RRFS_v1beta" ) + "FV3_RRFS_v1beta" | \ + "FV3_HRRR" ) if [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ]; then varmap_file="GSDphys_var_map.txt" diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 873fab97d1..04f4dd0bc3 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -315,6 +315,22 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}" # #----------------------------------------------------------------------- # +# Copy the two orography files needed for the drag suite in the FV3_HRRR +# physics suite. +# +# Note that the following is a temporary fix. We need a long-term solution +# that calls a script or program to generates the necessary files (instead +# of copying them). +# +#----------------------------------------------------------------------- +# +if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then + cp_vrfy ${GWD_HRRRsuite_DIR}/${CRES}*_ls.*.nc ${OROG_DIR} + cp_vrfy ${GWD_HRRRsuite_DIR}/${CRES}*_ss.*.nc ${OROG_DIR} +fi +# +#----------------------------------------------------------------------- +# # Note that the orography filtering code assumes that the regional grid # is a GFDLgrid type of grid; it is not designed to handle ESGgrid type # regional grids. If the flag "regional" in the orography filtering diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 7f78b4cf64..e712fb2b57 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -282,6 +282,33 @@ Cannot create symlink because target does not exist: fi +# +# If using the FV3_HRRR physics suite, there are two files (that contain +# statistics of the orography) that are needed by the gravity wave drag +# parameterization in that suite. Below, create symlinks to these files +# in the run directory. Note that the symlinks must have specific names +# that the FV3 model is hardcoded to recognize, and those are the names +# we use below. +# +if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then + + fileids=( "ss" "ls" ) + for fileid in "${fileids[@]}"; do + target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data_${fileid}.tile${TILE_RGNL}.halo${NH0}.nc" + symlink="oro_data_${fileid}.nc" + if [ -f "${target}" ]; then + ln_vrfy -sf ${relative_or_null} $target $symlink + else + print_err_msg_exit "\ +Cannot create symlink because target does not exist: + target = \"${target}\" + symlink = \"${symlink}\"" + fi + done + +fi + + # #----------------------------------------------------------------------- # diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 417583bc5c..a97b4330fe 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -8,8 +8,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" FCST_LEN_HRS="06" LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..5cb5950e34 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..45975fa2b3 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="RAP" +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh new file mode 100644 index 0000000000..1d63e8bd31 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GSD_SAR" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +#USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 992765165c..9308bd919e 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -8,8 +8,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" FCST_LEN_HRS="06" LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..2ba53af663 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 69677f8caf..9c2a9a5a0f 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -8,8 +8,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" FCST_LEN_HRS="06" LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..cfeb6faecd --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 8b1df05aa9..819833ed80 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -27,16 +27,21 @@ grid_RRFS_AK_3km_RAP_RAP grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS grid_RRFS_SUBCONUS_3km_HRRR_RAP diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 47ac5b4777..3f53aad581 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -554,6 +554,41 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # #----------------------------------------------------------------------- # +# If using the FV3_HRRR physics suite, set the base directory in which +# the pregenerated orography statistics files needed by the gravity wave +# drag parameterization in this suite are located. +# +#----------------------------------------------------------------------- +# + if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then + + if [ "$MACHINE" = "HERA" ]; then + GWD_HRRRsuite_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" + elif [ "$MACHINE" = "JET" ]; then + GWD_HRRRsuite_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" + elif [ "$MACHINE" = "CHEYENNE" ]; then + GWD_HRRRsuite_BASEDIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" + else + print_err_msg_exit "\ +The base directory (GWD_HRRRsuite_BASEDIR) containing the pregenerated +orography statistics files needed by the gravity wave drag parameterization +in the FV3_HRRR physics suite has not been specified for this machine +(MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + str=${str}" +# +# Base directory containing the pregenerated orography statistics files +# needed by the gravity wave drag parameterization in the FV3_HRRR physics +# suite. +# +GWD_HRRRsuite_BASEDIR=\"${GWD_HRRRsuite_BASEDIR}\"" + + fi +# +#----------------------------------------------------------------------- +# # # #----------------------------------------------------------------------- @@ -610,7 +645,7 @@ envir=\"\${EXPT_SUBDIR}\"" if [ "$MACHINE" = "HERA" ]; then FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then - FIXLAM_NCO_BASEDIR="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/FV3LAM_pregen" + FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" elif [ "$MACHINE" = "CHEYENNE" ]; then FIXLAM_NCO_BASEDIR="/needs/to/be/specified" else diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 0234ef47a6..7618e3aa29 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1374,3 +1374,24 @@ FVCOM_FILE="fvcom.nc" #------------------------------------------------------------------------ # COMPILER="intel" +# +#----------------------------------------------------------------------- +# +# GWD_HRRRsuite_BASEDIR: +# Temporary workflow variable specifies the base directory in which to +# look for certain fixed orography statistics files needed only by the +# gravity wave drag parameterization in the FV3_HRRR physics suite. This +# variable is added in order to avoid including hard-coded paths in the +# workflow scripts. Currently, the workflow simply copies the necessary +# files from a subdirectory under this directory (named according to the +# specified predefined grid) to the orography directory (OROG_DIR) under +# the experiment directory. +# +# Note that this variable is only used when using the FV3_HRRR physics +# suite. It should be removed from the workflow once there is a script +# or code available that generates these files for any grid. +# +#----------------------------------------------------------------------- +# +GWD_HRRRsuite_BASEDIR="" + diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 03cc137dc0..1d1cba5279 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -531,6 +531,7 @@ fi # # NOTE: # May want to remove lsoil from FV3.input.yml (and maybe input.nml.FV3). +# Also, may want to set lsm here as well depending on SDF_USES_RUC_LSM. # lsoil="4" if [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" -o \ diff --git a/ush/link_fix.sh b/ush/link_fix.sh index 651f4fbfd0..29ce0ecc1d 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -245,6 +245,12 @@ Creating links in the FIXLAM directory to the grid files..." "C*${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc" \ "C*${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" \ ) + if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then + fns+=( \ + "C*${DOT_OR_USCORE}oro_data_ss.tile${TILE_RGNL}.halo${NH0}.nc" \ + "C*${DOT_OR_USCORE}oro_data_ls.tile${TILE_RGNL}.halo${NH0}.nc" \ + ) + fi fps=( "${fns[@]/#/${OROG_DIR}/}" ) run_task="${RUN_TASK_MAKE_OROG}" ;; diff --git a/ush/setup.sh b/ush/setup.sh index aa72485617..64a74b3140 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -977,6 +977,39 @@ fi # #----------------------------------------------------------------------- # +# If using the FV3_HRRR physics suite, make sure that the directory from +# which certain fixed orography files will be copied to the experiment +# directory actually exists. Note that this is temporary code. It should +# be removed once there is a script or code available that will create +# these orography files for any grid. +# +#----------------------------------------------------------------------- +# +GWD_HRRRsuite_DIR="${GWD_HRRRsuite_BASEDIR}/${PREDEF_GRID_NAME}" +if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then + if [ -z "${PREDEF_GRID_NAME}" ]; then + print_err_msg_exit "\ +A predefined grid name (PREDEF_GRID_NAME) must be specified when using +the FV3_HRRR physics suite: + CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" + PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\"" + else + if [ ! -d "${GWD_HRRRsuite_DIR}" ]; then + print_err_msg_exit "\ +The directory (GWD_HRRRsuite_DIR) that should contain the gravity wave +drag-related orography files for the FV3_HRRR suite does not exist: + GWD_HRRRsuite_DIR = \"${GWD_HRRRsuite_DIR}\"" + elif [ ! "$( ls -A ${GWD_HRRRsuite_DIR} )" ]; then + print_err_msg_exit "\ +The directory (GWD_HRRRsuite_DIR) that should contain the gravity wave +drag related orography files for the FV3_HRRR suite is empty: + GWD_HRRRsuite_DIR = \"${GWD_HRRRsuite_DIR}\"" + fi + fi +fi +# +#----------------------------------------------------------------------- +# # If the base directory (EXPT_BASEDIR) in which the experiment subdirectory # (EXPT_SUBDIR) will be located does not start with a "/", then it is # either set to a null string or contains a relative directory. In both @@ -2336,6 +2369,7 @@ CYCLE_BASEDIR="${CYCLE_BASEDIR}" GRID_DIR="${GRID_DIR}" OROG_DIR="${OROG_DIR}" SFC_CLIMO_DIR="${SFC_CLIMO_DIR}" +GWD_HRRRsuite_DIR="${GWD_HRRRsuite_DIR}" NDIGITS_ENSMEM_NAMES="${NDIGITS_ENSMEM_NAMES}" ENSMEM_NAMES=( $( printf "\"%s\" " "${ENSMEM_NAMES[@]}" )) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index ac27188b71..ea728485e3 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -16,6 +16,7 @@ FV3_GSD_v0: FV3_GSD_SAR: gfs_physics_nml: &gsd_sar_phys + do_deep: False effr_in: True imfdeepcnv: 0 imfshalcnv: 0 @@ -31,7 +32,6 @@ FV3_GSD_SAR: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - do_deep: False FV3_RRFS_v1alpha: gfs_physics_nml: @@ -59,6 +59,33 @@ FV3_RRFS_v1beta: atmos_model_nml: fdiag: 3 +FV3_HRRR: + fv_core_nml: + hord_dp: -5 + hord_mt: 5 + hord_tm: 5 + hord_vt: 5 + kord_mt: 9 + kord_tm: -9 + kord_tr: 9 + kord_wz: 9 + nord_tr: 2 + nrows_blend: 10 + regional_bcs_from_gsi: False + write_restart_with_bcs: False + gfs_physics_nml: + <<: *gsd_sar_phys + cdmbgwd: [3.5, 1.0] + do_mynnsfclay: True + do_sfcperts: !!python/none + effr_in: True + gwd_opt: 3 + iaer: 5111 + icliq_sw: 2 + imfdeepcnv: -1 + imfshalcnv: -1 + iovr: 3 + FV3_GFS_2017_gfdlmp: atmos_model_nml: avg_max_length: 3600.0 diff --git a/ush/templates/diag_table.FV3_HRRR b/ush/templates/diag_table.FV3_HRRR new file mode 100644 index 0000000000..1482af2aa3 --- /dev/null +++ b/ush/templates/diag_table.FV3_HRRR @@ -0,0 +1,361 @@ +{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional +{{ starttime.strftime("%Y %m %d %H %M %S") }} + +"grid_spec", -1, "months", 1, "days", "time" +"atmos_static", -1, "hours", 1, "hours", "time" +#"atmos_4xdaily", 1, "hours", 1, "days", "time" +"fv3_history", 1, "years", 1, "hours", "time" +"fv3_history2d", 1, "years", 1, "hours", "time" + +# +#======================= +# ATMOSPHERE DIAGNOSTICS +#======================= +### +# grid_spec +### + "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, + "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, +### +# 4x daily output +### +# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 +### +# gfs static data +### + "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 + "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 + "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 + "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 + "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 +### +# FV3 variabls needed for NGGPS evaluation +### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 + +"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 + +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt5", "soilt5" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt6", "soilt6" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt7", "soilt7" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt8", "soilt8" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt9", "soilt9" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw5", "soilw5" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw6", "soilw6" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw7", "soilw7" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw8", "soilw8" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw9", "soilw9" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_5", "soill5", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_6", "soill6", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_7", "soill7", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_8", "soill8", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_9", "soill9", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +# Aerosols (CCN, IN) from Thompson microphysics +"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 +# Cloud effective radii from Thompson and WSM6 microphysics +"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 +# Prognostic/diagnostic variables from MYNN +"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 +#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 +# Prognostic/diagnostic variables from RUC LSM +"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +# Stochastic physics +"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 +"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/field_table.FV3_HRRR b/ush/templates/field_table.FV3_HRRR new file mode 100644 index 0000000000..0a927de455 --- /dev/null +++ b/ush/templates/field_table.FV3_HRRR @@ -0,0 +1,65 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic graupel mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud water number concentration + "TRACER", "atmos_mod", "water_nc" + "longname", "cloud liquid water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# water- and ice-friendly aerosols (Thompson) + "TRACER", "atmos_mod", "liq_aero" + "longname", "water-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / + "TRACER", "atmos_mod", "ice_aero" + "longname", "ice-friendly aerosol number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=1.e30" / diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index f94a1569d2..41a73690fb 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -31,6 +31,7 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GFS_v16beta" \ "FV3_RRFS_v1beta" \ "FV3_RRFS_v1alpha" \ +"FV3_HRRR" \ ) valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") From 52335ba38ffac7880291a9fd7217c285489a0b36 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 28 Jan 2021 12:27:08 -0700 Subject: [PATCH 016/203] develop: add support for SPP. (#409) * Add default namelist with SPP entries. * Changes necessary to run with SPP * Typo fix in generate script. * Changes to implement SPP. * Add comment regarding use of SPP. * Pass N_VAR_SPP to the var_defns.sh file. --- ush/config_defaults.sh | 26 +++++++++++++++++++- ush/generate_FV3LAM_wflow.sh | 11 +++++++++ ush/set_FV3nml_stoch_params.sh | 5 ++++ ush/setup.sh | 43 +++++++++++++++++++++++++++++++++- ush/templates/input.nml.FV3 | 13 ++++++++++ ush/valid_param_vals.sh | 1 + 6 files changed, 97 insertions(+), 2 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 7618e3aa29..36bdff562f 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1307,7 +1307,7 @@ NUM_ENS_MEMBERS="1" # #----------------------------------------------------------------------- # -# Set default stochastic physics options +# Set default ad-hoc stochastic physics options. # For detailed documentation of these parameters, see: # https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html # @@ -1332,6 +1332,30 @@ SKEB_VDOF="10" USE_ZMTNBLCK="false" # #----------------------------------------------------------------------- +# +# Set default SPP stochastic physics options. +# Each SPP option is an array, applicable (in order) to the scheme/parameter +# listed in SPP_VAR_LIST. Enter each value of the array in config.sh as +# shown below without commas or single quotes (e.g., SPP_VAR_LIST= +# ( "pbl" "lsm" "mp" ). Both commas and single quotes will be added by +# Jinja when creating the namelist. +# +# Note that SPP is currently only available for specific physics schemes +# used in the RAP/HRRR physics suite. Users need to be aware of which SDF +# is chosen when turning this option on. +# +#----------------------------------------------------------------------- +# +DO_SPP="false" +SPP_VAR_LIST=( "pbl" ) +SPP_MAG_LIST=( "0.2" ) #Variable "spp_prt_list" in input.nml +SPP_LSCALE=( "150000.0" ) +SPP_TSCALE=( "21600.0" ) #Variable "spp_tau" in input.nml +SPP_SIGTOP1=( "0.1" ) +SPP_SIGTOP2=( "0.025" ) +SPP_STDDEV_CUTOFF=( "1.5" ) +# +#----------------------------------------------------------------------- # # HALO_BLEND: # Number of rows into the computational domain that should be blended diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 1d1cba5279..3be181c12a 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -586,6 +586,17 @@ settings="\ 'do_shum': ${DO_SHUM}, 'do_sppt': ${DO_SPPT}, 'do_skeb': ${DO_SKEB}, + 'do_spp': ${DO_SPP}, + 'n_var_spp': ${N_VAR_SPP}, + } +'nam_spperts': { + 'spp_lscale': [ $( printf %s, "${SPP_LSCALE[@]}" ) ], + 'spp_prt_list': [ $( printf %s, "${SPP_MAG_LIST[@]}" ) ], + 'spp_sigtop1': [ $( printf %s, "${SPP_SIGTOP1[@]}" ) ], + 'spp_sigtop2': [ $( printf %s, "${SPP_SIGTOP2[@]}" ) ], + 'spp_stddev_cutoff': [ $( printf %s, "${SPP_STDDEV_CUTOFF[@]}" ) ], + 'spp_tau': [ $( printf %s, "${SPP_TSCALE[@]}" ) ], + 'spp_var_list': [ $( printf %s, "${SPP_VAR_LIST[@]}" ) ], } 'nam_stochy': { 'shum': ${SHUM_MAG}, diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index 98e1c34691..3f401ece70 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -79,6 +79,7 @@ function set_FV3nml_stoch_params() { iseed_shum \ iseed_skeb \ iseed_sppt \ + iseed_spp \ settings # #----------------------------------------------------------------------- @@ -120,12 +121,16 @@ for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do iseed_shum=$(( cdate*1000 + ip1*10 + 2 )) iseed_skeb=$(( cdate*1000 + ip1*10 + 3 )) iseed_sppt=$(( cdate*1000 + ip1*10 + 1 )) + iseed_spp=$(( cdate*1000 + ip1*10 + 4 )) settings="\ 'nam_stochy': { 'iseed_shum': ${iseed_shum}, 'iseed_skeb': ${iseed_skeb}, 'iseed_sppt': ${iseed_sppt}, + } +'nam_spperts': { + 'iseed_spp': ${iseed_spp}, }" $USHDIR/set_namelist.py -q \ diff --git a/ush/setup.sh b/ush/setup.sh index 64a74b3140..8b08d32548 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -292,6 +292,26 @@ fi # #----------------------------------------------------------------------- # +# Make sure that DO_SPP is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" +# +# Set DO_SPP to either "TRUE" or "FALSE" so we don't +# have to consider other valid values later on. +# +DO_SPP=${DO_SPP^^} +if [ "${DO_SPP}" = "TRUE" ] || \ + [ "${DO_SPP}" = "YES" ]; then + DO_SPP="TRUE" +elif [ "${DO_SPP}" = "FALSE" ] || \ + [ "${DO_SPP}" = "NO" ]; then + DO_SPP="FALSE" +fi +# +#----------------------------------------------------------------------- +# # Set magnitude of stochastic ad-hoc schemes to -999.0 if they are not # being used. This is required at the moment, since "do_shum/sppt/skeb" # does not override the use of the scheme unless the magnitude is also @@ -312,6 +332,18 @@ fi # #----------------------------------------------------------------------- # +# If running with SPP, count the number of entries in SPP_VAR_LIST to +# correctly set N_VAR_SPP, otherwise set it to zero. +# +#----------------------------------------------------------------------- +# +N_VAR_SPP=0 +if [ "${DO_SPP}" = "TRUE" ]; then + N_VAR_SPP=${#SPP_VAR_LIST[@]} +fi +# +#----------------------------------------------------------------------- +# # Make sure that USE_FVCOM is set to a valid value and assign directory # and file names. # @@ -2518,7 +2550,7 @@ fi # #----------------------------------------------------------------------- # -# Continue appending variable defintions to the variable definitions +# Continue appending variable definitions to the variable definitions # file. # #----------------------------------------------------------------------- @@ -2608,6 +2640,15 @@ FVCOM_FILE="${FVCOM_FILE}" # NCORES_PER_NODE="${NCORES_PER_NODE}" PE_MEMBER01="${PE_MEMBER01}" +# +#----------------------------------------------------------------------- +# +# IF DO_SPP="TRUE," N_VAR_SPP is the number of parameterizations that +# are perturbed with SPP, otherwise N_VAR_SPP=0. +# +#----------------------------------------------------------------------- +# +N_VAR_SPP="${N_VAR_SPP}" EOM } || print_err_msg_exit "\ Heredoc (cat) command to append new variable definitions to variable diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 64f9f4d05d..decc418583 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -140,6 +140,7 @@ do_mynnsfclay = .false. do_shum = .false. do_skeb = .false. + do_spp = .false. do_sppt = .false. dspheat = .true. fhcyc = 0.0 @@ -173,6 +174,7 @@ ltaerosol = .true. lwhtr = .true. n_var_lndp = 0 + n_var_spp = 0 ncld = 5 nsradar_reset = 3600 nst_anl = .true. @@ -205,6 +207,17 @@ lndp_prt_list = 0.2,0.1, / +&nam_spperts + iseed_spp = 4 + spp_lscale = 150000.0 + spp_prt_list = 0.2 + spp_sigtop1 = 0.1 + spp_sigtop2 = 0.025 + spp_stddev_cutoff = 1.5 + spp_tau = 21600.0 + spp_var_list = '' +/ + &nam_stochy iseed_shum = 2 iseed_skeb = 3 diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 41a73690fb..d05591b0fb 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -56,6 +56,7 @@ valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CUSTOM_POST_CONFIG_FILE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From ec125d9848fa63cb6469179143cabd9fe9fbc7ac Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Tue, 2 Feb 2021 20:47:27 -0700 Subject: [PATCH 017/203] Remove SPP from input.nml.FV3 template until code is ready in ufs-weather-model. (#420) * Remove SPP lines from the input.nml.FV3 template until code is ready in ufs-weather-model * Remote references to SPP in the generate script. * Remove references to SPP in the generate script. --- ush/generate_FV3LAM_wflow.sh | 11 ----------- ush/templates/input.nml.FV3 | 13 ------------- 2 files changed, 24 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 3be181c12a..1d1cba5279 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -586,17 +586,6 @@ settings="\ 'do_shum': ${DO_SHUM}, 'do_sppt': ${DO_SPPT}, 'do_skeb': ${DO_SKEB}, - 'do_spp': ${DO_SPP}, - 'n_var_spp': ${N_VAR_SPP}, - } -'nam_spperts': { - 'spp_lscale': [ $( printf %s, "${SPP_LSCALE[@]}" ) ], - 'spp_prt_list': [ $( printf %s, "${SPP_MAG_LIST[@]}" ) ], - 'spp_sigtop1': [ $( printf %s, "${SPP_SIGTOP1[@]}" ) ], - 'spp_sigtop2': [ $( printf %s, "${SPP_SIGTOP2[@]}" ) ], - 'spp_stddev_cutoff': [ $( printf %s, "${SPP_STDDEV_CUTOFF[@]}" ) ], - 'spp_tau': [ $( printf %s, "${SPP_TSCALE[@]}" ) ], - 'spp_var_list': [ $( printf %s, "${SPP_VAR_LIST[@]}" ) ], } 'nam_stochy': { 'shum': ${SHUM_MAG}, diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index decc418583..64f9f4d05d 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -140,7 +140,6 @@ do_mynnsfclay = .false. do_shum = .false. do_skeb = .false. - do_spp = .false. do_sppt = .false. dspheat = .true. fhcyc = 0.0 @@ -174,7 +173,6 @@ ltaerosol = .true. lwhtr = .true. n_var_lndp = 0 - n_var_spp = 0 ncld = 5 nsradar_reset = 3600 nst_anl = .true. @@ -207,17 +205,6 @@ lndp_prt_list = 0.2,0.1, / -&nam_spperts - iseed_spp = 4 - spp_lscale = 150000.0 - spp_prt_list = 0.2 - spp_sigtop1 = 0.1 - spp_sigtop2 = 0.025 - spp_stddev_cutoff = 1.5 - spp_tau = 21600.0 - spp_var_list = '' -/ - &nam_stochy iseed_shum = 2 iseed_skeb = 3 From 9b83f82ceedd108e83f6bbff26f4037fe1b6aa51 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 3 Feb 2021 09:26:06 -0700 Subject: [PATCH 018/203] Improvements to run workflow with NAM ICs/LBCs (#417) ## DESCRIPTION OF CHANGES: * In set_extrn_mdl_params.sh, add stanzas for the "NAM" external model as well as for the "CHEYENNE" platform. * In run_experiments.sh, add stanzas for "NAM". * WE2E tests: * Add 3 new WE2E tests for the NAM on the RRFS_CONUS_25km grid. These use NAM for both the ICs and LBCs, run 24hr forecasts, and use an LBC update interval of 3 hours. They all run the 2015060212 case, which is currently the only date for which NAM data is available (provided by Bill Gallus and Jonathan Thielen). The difference between the three tests is that they run the FV3_GSD_SAR, FV3_HRRR, and FV3_RRFS_v1beta suites, respectively. * Modify the four WE2E tests that run on the RRFS_CONUS_25km grid with HRRR/RAP ICs/LBCs so that they all use the 2020 Derecho case (2020081000) and go out to 24 hours (with a boundary update every 3 hours). This is because the Derecho case is of more interest to the community than the (randomly chosen) previous case 2020080100. The four tests run using the GSD_SAR, HRRR, RRFS_v1alpha, and RRFS_v1beta suites, respectively. * For comparison purposes between the ICs/LBCs combinations of NAM/NAM, HRRR/RAP, and HRRR/HRRR, add 2 WE2E tests on the RRFS_COUNS_25km grid that use HRRR for both ICs and LBCs (and run the 2020 Derecho case for 24 hrs with a 3-hour boundary update). * Modify script that gets experiments' workflow status so that it ignores all non-experiment directories as well as all "inactive" (i.e. renamed) experiment directories under the specified experiments base directory. * Modify the default value of DT_ATMOS on the RRFS_CONUS_25km grid to be 40sec (due to test results below). ## TESTS CONDUCTED: On Cheyenne, ran the following tests on the RRFS_CONUS_25km grid with both DT_ATMOS=300sec and DT_ATMOS=40sec. The 40sec tests all succeeded (although some with error messages) while some of the 300sec tests succeeded while others failed. Full results are as follows: * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta * DT_ATMOS = 300sec: succeeded * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR (CDATE = 2015060212) * DT_ATMOS = 300sec: failed * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR (CDATE = 2015060212) * DT_ATMOS = 300sec: failed * DT_ATMOS = 40sec: succeeded * grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta (CDATE = 2015060212) * DT_ATMOS = 300sec: failed * DT_ATMOS = 40sec: succeeded It is not yet clear what causes the failures with the NAM experiments, but since these succeed with DT_ATMOS=40sec, we change the default value of DT_ATMOS to 40 sec for the RRFS_CONUS_25km grid. ## CONTRIBUTORS (optional): Bill Gallus --- ...S_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh | 2 +- ...ONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh | 17 ++ ...km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh | 17 ++ ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 2 +- ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 6 +- ...5km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 6 +- ...NUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh | 17 ++ ..._CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh | 17 ++ ...25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh | 17 ++ tests/baselines_list.txt | 5 + tests/get_expts_status.sh | 206 +++++++++++++++--- tests/run_experiments.sh | 4 + ush/set_extrn_mdl_params.sh | 28 +++ ush/set_predef_grid_params.sh | 2 +- 15 files changed, 308 insertions(+), 40 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh index 1d63e8bd31..f1d559b1a5 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh @@ -14,4 +14,4 @@ CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" -#USE_USER_STAGED_EXTRN_FILES="TRUE" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh new file mode 100644 index 0000000000..fa2001d51a --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..af5a5aec07 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 9308bd919e..4241d9ad59 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -5,7 +5,7 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" +FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="3" DATE_FIRST_CYCL="20200810" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 2ba53af663..5b07c80f31 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -5,7 +5,7 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" +FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="1" DATE_FIRST_CYCL="20200810" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index e517b6e3b6..ccb8ff6862 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -6,11 +6,11 @@ GRID_GEN_METHOD="ESGgrid" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" -FCST_LEN_HRS="06" +FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 330e1eca5f..64248d44b5 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -5,11 +5,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="06" +FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh new file mode 100644 index 0000000000..53f6b27774 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GSD_SAR" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20150602" +DATE_LAST_CYCL="20150602" +CYCL_HRS=( "12" ) + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh new file mode 100644 index 0000000000..42305f3afe --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20150602" +DATE_LAST_CYCL="20150602" +CYCL_HRS=( "12" ) + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..81e9fc0bb6 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20150602" +DATE_LAST_CYCL="20150602" +CYCL_HRS=( "12" ) + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 819833ed80..9ba6347940 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -34,9 +34,14 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR +grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta diff --git a/tests/get_expts_status.sh b/tests/get_expts_status.sh index 3b65422e3e..5c05acafb1 100755 --- a/tests/get_expts_status.sh +++ b/tests/get_expts_status.sh @@ -3,11 +3,19 @@ # #----------------------------------------------------------------------- # -# This script checks the workflow status of all forecast experiments -# with directories located under a given base directory (expts_subdir). -# It must be supplied exactly one argument -- the experiments base -# directory. It assumes that all subdirectories under this base directory -# are experiment directories and checks each one's workflow status. +# This script updates and reports back the workflow status of all active +# forecast experiments under a specified base directory (expts_basedir). +# It must be supplied exactly one argument, which is the full path to the +# experiments base directory. +# +# The script first determines which of the subdirectories under the base +# directory represent active experiments (see below for how this is done). +# For all such experiments, it calls the workflow (re)launch script to +# update the status of the workflow and prints the status out to screen. +# It also generates a summary status file in the base directory that +# contains the last num_tail_lines lines (defined below) of each experiment's +# workflow log file [which is generated by the (re)launch script] and thus +# has information on which tasks may have succeeded/failed]. # #----------------------------------------------------------------------- # @@ -109,31 +117,167 @@ fi #----------------------------------------------------------------------- # cd "${expts_basedir}" +# # Get a list of all subdirectories (but not files) in the experiment base # directory. Note that the ls command below will return a string containing # the subdirectory names, with each name followed by a backslash and a # newline. -expts_list=$( \ls -1 -d */ ) +# +subdirs_list=$( \ls -1 -d */ ) +# # Remove all backslashes from the ends of the subdirectory names. -expts_list=$( printf "${expts_list}" "%s" | sed -r 's|/||g' ) +# +subdirs_list=$( printf "${subdirs_list}" "%s" | sed -r 's|/||g' ) +# # Create an array out of the string containing the newline-separated list -# of experiment subdirectories. -expts_list=( ${expts_list} ) +# of subdirectories. +# +subdirs_list=( ${subdirs_list} ) # #----------------------------------------------------------------------- # -# Get the number of experiments for which to check the workflow status -# and print out an informational message. +# Loop through the elements of the array subdirs_list and create an array +# containing a list of all active experiment subdirectories under the +# experiment base directory. These active subdirectories will be further +# processed later below. Here, by "active" experiment subdirectory, we +# mean a subdirectory that (1) contains a forecast experiment (i.e. was +# was created by the experiment generation scripts) and (2) does not +# represent an old experiment whose workflow status is no longer relevant. +# For this purpose, for each element in subdirs_list, we: +# +# 1) Change location to the subdirectory. +# +# 2) Check whether an experiment variable definitions file (var_defns.sh) +# exists. If so, we assume the subdirectory is an experiment directory. +# If not, we assume it is not, in which case the subdirectory will +# not be added to the list of active experiment subdirectories. +# +# 3) If the subdirectory is an experiment directory, ensure that it is +# an active experiment, i.e. that it is not an old experiment that +# has been renamed and whose experiment status is thus irrelevant. +# For this purpose, we source the variable definitions file in order +# to have available the workflow variable EXPT_SUBDIR that contains +# the name of the experiment when it was first created. If this +# matches the name of the current subdirectory, then add the latter +# to the list of active experiment subdirectories; otherwise, do not. +# In the latter case, we are assuming that the original experiment +# subdirectory was renamed (e.g. to something like the orginal name +# with the string "_old001" appended) and thus does not contain an +# active experiment whose workflow status is of interest. +# +# 4) Change location back to the experiments base directory. # #----------------------------------------------------------------------- # -num_expts="${#expts_list[@]}" -expts_list_str=$( printf " \'%s\'\n" "${expts_list[@]}" ) +separator="======================================" + +var_defns_fn="var_defns.sh" +j="0" +expt_subdirs=() + +num_subdirs="${#subdirs_list[@]}" +for (( i=0; i<=$((num_subdirs-1)); i++ )); do + + subdir="${subdirs_list[$i]}" + msg=" +$separator +Checking whether the subdirectory + \"${subdir}\" +contains an active experiment..." + print_info_msg "$msg" + + cd_vrfy "${subdir}" +# +# If a variable definitions file does not exist, print out a message +# and move on to the next subdirectory. +# + if [ ! -f "${var_defns_fn}" ]; then + + print_info_msg " +The current subdirectory (subdir) under the experiments base directory +(expts_basedir) does not contain an experiment variable defintions file +(var_defns_fn): + expts_basedir = \"${expts_basedir}\" + subdir = \"${subdir}\" + var_defns_fn = \"${var_defns_fn}\" +Thus, we will assume it is not an experiment directory and will not add +it to the list of active experiments subdirectories whose workflow status +must be checked." +# +# If a variable definitions file does exist, then... +# + else +# +# Source the variable definitions file. +# + . "./${var_defns_fn}" +# +# If the workflow variable EXPT_SUBDIR is the same as the name of the +# current subdirectory, then assume this subdirectory contains an active +# experiment. In this case, print out a message and add its name to the +# list of such experiments. +# + if [ "${EXPT_SUBDIR}" = "$subdir" ]; then + + print_info_msg " +The current subdirectory (subdir) under the experiments base directory +(expts_basedir) contains an active experiment: + expts_basedir = \"${expts_basedir}\" + subdir = \"${subdir}\" +Adding the current subdirectory to the list of active experiment +subdirectories whose workflow status must be checked." + + expt_subdirs[$j]="$subdir" + j=$((j+1)) +# +# If the workflow variable EXPT_SUBDIR is not the same as the name of +# the current subdirectory, then assume this subdirectory contains an +# "inactive" that has been renamed. In this case, print out a message +# and move on to the next subdirectory (whithout adding the the name of +# the currend subdirectory to the list of active experiments). +# + else + + print_info_msg " +The current subdirectory (subdir) under the experiments base directory +(expts_basedir) contains an experiment whose original name (EXPT_SUBDIR) +does not match the name of the current subdirectory: + expts_basedir = \"${expts_basedir}\" + subdir = \"${subdir}\" + EXPT_SUBDIR = \"${EXPT_SUBDIR}\" +Thus, we will assume that the current subdirectory contains an inactive +(i.e. old) experiment whose workflow status is not relevant and will not +add it to the list of active experiment subdirectories whose workflow +status must be checked." + + fi + + fi + + print_info_msg "\ +$separator +" +# +# Change location back to the experiments base directory. +# + cd_vrfy "${expts_basedir}" + +done +# +#----------------------------------------------------------------------- +# +# Get the number of active experiments for which to check the workflow +# status and print out an informational message. +# +#----------------------------------------------------------------------- +# +num_expts="${#expt_subdirs[@]}" +expt_subdirs_str=$( printf " \'%s\'\n" "${expt_subdirs[@]}" ) print_info_msg " -The number of experiments found is: +The number of active experiments found is: num_expts = ${num_expts} The list of experiments whose workflow status will be checked is: -${expts_list_str} +${expt_subdirs_str} " # #----------------------------------------------------------------------- @@ -155,31 +299,31 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename" # #----------------------------------------------------------------------- # -# Loop through the elements of the array expts_list. For each element -# (i.e. for each experiment), change location to the experiment directory -# and call the script launch_FV3LAM_wflow.sh to update the log file -# log.launch_FV3LAM_wflow. Then take the last num_tail_lines of this -# log file (along with an appropriate message) and add it to the status -# report file. +# Loop through the elements of the array expt_subdirs. For each element +# (i.e. for each active experiment), change location to the experiment +# directory and call the script launch_FV3LAM_wflow.sh to update the log +# file log.launch_FV3LAM_wflow. Then take the last num_tail_lines of +# this log file (along with an appropriate message) and add it to the +# status report file. # #----------------------------------------------------------------------- # +launch_wflow_fn="launch_FV3LAM_wflow.sh" num_tail_lines="40" for (( i=0; i<=$((num_expts-1)); i++ )); do - expt_subdir="${expts_list[$i]}" - msg=" -====================================== + expt_subdir="${expt_subdirs[$i]}" + print_info_msg "\ +$separator Checking workflow status of experiment: \"${expt_subdir}\"" - print_info_msg "$msg" # # Change location to the experiment subdirectory, call the workflow launch # script to update the launch log file, and capture the output from that # call. # cd_vrfy "${expt_subdir}" - launch_msg=$( launch_FV3LAM_wflow.sh 2>&1 ) + launch_msg=$( "${launch_wflow_fn}" 2>&1 ) log_tail=$( tail -n ${num_tail_lines} log.launch_FV3LAM_wflow ) # # Record the tail from the log file into the status report file. @@ -188,14 +332,16 @@ Checking workflow status of experiment: \"${expt_subdir}\"" print_info_msg "${log_tail}" >> "${expts_status_fp}" # # Print the workflow status to the screen. +# wflow_status=$( printf "${log_tail}" | grep "Workflow status:" ) # wflow_status="${wflow_status## }" # Not sure why this doesn't work to strip leading spaces. wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) - msg="\ -${wflow_status} -====================================== -" + msg="${wflow_status}" print_info_msg "$msg" + + print_info_msg "\ +$separator +" # # Change location back to the experiments base directory. # diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 3f53aad581..5bc3e11411 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -751,6 +751,8 @@ machine (MACHINE): elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) + elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then + EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" @@ -781,6 +783,8 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) + elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) fi str=${str}" diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index cd63a1ff4e..891c70e6d5 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -143,6 +143,17 @@ else esac ;; + "NAM") + case $MACHINE in + "HERA") + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" + ;; + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" + ;; + esac + ;; + esac fi @@ -181,6 +192,9 @@ case ${EXTRN_MDL_NAME_LBCS} in "HRRR") EXTRN_MDL_LBCS_OFFSET_HRS="0" ;; + "NAM") + EXTRN_MDL_LBCS_OFFSET_HRS="0" + ;; esac # #----------------------------------------------------------------------- @@ -291,6 +305,20 @@ else "JET") EXTRN_MDL_SYSBASEDIR_LBCS="/misc/whome/rtrr/hrrr" ;; + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" + ;; + esac + ;; + + "NAM") + case $MACHINE in + "HERA") + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" + ;; + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" + ;; esac ;; diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index e87e9ecdb5..f01ab6d306 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -97,7 +97,7 @@ case ${PREDEF_GRID_NAME} in ESGgrid_WIDE_HALO_WIDTH="6" - DT_ATMOS="${DT_ATMOS:-300}" + DT_ATMOS="${DT_ATMOS:-40}" LAYOUT_X="${LAYOUT_X:-5}" LAYOUT_Y="${LAYOUT_Y:-2}" From 3702c855c4a31c8208d0726bd7ed7bcdf7f76a52 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 3 Feb 2021 13:11:15 -0700 Subject: [PATCH 019/203] Bug fix for setting GWD_HRRRsuite_BASEDIR when running the FV3_HRRR suite in NCO mode (#421) @chunhuazhou (Chunhua Zhou) reported that GWD_HRRRsuite_BASEDIR is not getting set correctly when running the FV3_HRRR suite in NCO mode. This PR is to fix that bug. ## DESCRIPTION OF CHANGES: ### Main changes: * In setup.sh, ensure that GWD_HRRRsuite_BASEDIR is set properly when running the FV3_HRRR suite in NCO mode, and add checks to make sure directories exist, etc. * Add 2 new WE2E tests for running the FV3_HRRR suite in NCO mode on the RRFS_CONUS_25km and RRFS_CONUS_3km grids. The test names are nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR and nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR. These run the 2020020800 case, which is a winter case that has a history of crashing. ### Improvements: * Remove the regional_workflow/env directory since it is unused. * Change the names of 5 of the existing NCO-mode WE2E tests to make it clearer what configurations they run (and to be consistent with the naming convention of community-mode tests). ## TESTS CONDUCTED: Ran the following WE2E tests (including the two new ones) on Hera. All were successful. * grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR * nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR * nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR (both on 2020020800 and 2020081006) --- env/JET.env | 167 --------------- env/THEIA.env | 167 --------------- env/WCOSS_C.env | 194 ----------------- env/WCOSS_DELL_P3.env | 200 ------------------ env/gfs.ver | 22 -- ...s_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh} | 0 ...S_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} | 0 ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 18 ++ ..._FV3GFS_suite_GFS_2017_gfdlmp_regional.sh} | 0 ...US_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} | 0 ..._CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 18 ++ ...US_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} | 0 tests/baselines_list.txt | 12 +- ush/config_defaults.sh | 2 +- ush/setup.sh | 54 ++++- 15 files changed, 97 insertions(+), 757 deletions(-) delete mode 100755 env/JET.env delete mode 100755 env/THEIA.env delete mode 100755 env/WCOSS_C.env delete mode 100755 env/WCOSS_DELL_P3.env delete mode 100644 env/gfs.ver rename tests/baseline_configs/{config.nco_CONUS_25km_GFDLgrid.sh => config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh} (100%) rename tests/baseline_configs/{config.nco_RRFS_CONUS_25km_HRRR_RAP.sh => config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} (100%) create mode 100644 tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename tests/baseline_configs/{config.nco_RRFS_CONUS_3km_FV3GFS_FV3GFS.sh => config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh} (100%) rename tests/baseline_configs/{config.nco_RRFS_CONUS_3km_HRRR_RAP.sh => config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} (100%) create mode 100644 tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename tests/baseline_configs/{config.nco_RRFS_SUBCONUS_3km_HRRR_RAP.sh => config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} (100%) diff --git a/env/JET.env b/env/JET.env deleted file mode 100755 index 5b14faf1a4..0000000000 --- a/env/JET.env +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Theia information -export npe_node_max=24 -export launcher="mpirun -np" - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 -#export LD_BIND_NOW=1 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Cray-CS400" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP}" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp" - -elif [ $step = "awips" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -elif [ $step = "gempak" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -fi diff --git a/env/THEIA.env b/env/THEIA.env deleted file mode 100755 index c747e1f99c..0000000000 --- a/env/THEIA.env +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Theia information -export npe_node_max=24 -export launcher="mpirun -np" - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export OMP_STACKSIZE=2048000 -export NTHSTACK=1024000000 -#export LD_BIND_NOW=1 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Cray-CS400" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP}" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp cfp" - -elif [ $step = "awips" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -elif [ $step = "gempak" ]; then - - echo "WARNING: $step is not enabled on $machine!" - -fi diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env deleted file mode 100755 index 506f187d82..0000000000 --- a/env/WCOSS_C.env +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# Cray information -export launcher="aprun" -export npe_node_max=24 - -# Configure MPI environment -export MPI_BUFS_PER_PROC=2048 -export MPI_BUFS_PER_HOST=2048 -export MPI_GROUP_MAX=256 -export MPI_MEMMAP_OFF=1 -export MP_STDOUTMODE="ORDERED" -export NTHSTACK=1024000000 -export OMP_STACKSIZE="2048M" -export KMP_AFFINITY="disabled" - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE=${POE:-"YES"} - export sys_tp="Cray-XC40" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_anal} -N $npe_node_anal -d $NTHREADS_GSI -cc depth" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher -j 1 -n \$ncmd -N 1 -d $NTHREADS_CALCINC -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - - export NTHREADS_CYCLE=${nth_cycle:-12} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_node_cycle=$(($npe_node_max / $NTHREADS_CYCLE)) - npe_cycle=${ntiles:-6} - [[ $npe_node_cycle -gt $npe_cycle ]] && npe_node_cycle=$npe_cycle - export APRUN_CYCLE="$launcher -j 1 -n $npe_cycle -N $npe_node_cycle -d $NTHREADS_CYCLE -cc depth" - - export NTHREADS_GAUSFCANL=1 - npe_node_gausfcanl=$(($npe_node_max / $NTHREADS_GAUSFCANL)) - npe_gausfcanl=${npe_gausfcanl:-1} - [[ $npe_node_gausfcanl -gt $npe_gausfcanl ]] && npe_node_gausfcanl=$npe_gausfcanl - export APRUN_GAUSFCANL="$launcher -j 1 -n $npe_gausfcanl -N $npe_node_gausfcanl -d $NTHREADS_GAUSFCANL -cc depth" - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_gsi:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher -j 1 -n ${npe_gsi:-$npe_eobs} -N $npe_node_eobs -d $NTHREADS_GSI -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_enkf:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher -j 1 -n ${npe_enkf:-$npe_eupd} -N $npe_node_eupd -d $NTHREADS_ENKF -cc depth" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher -q -j 1 -n \$ncmd -N \$ncmd_max -d 1 cfp" - fi - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_FV3 -cc depth" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher -j 1 -n $LEVS -N $npe_node_fcst -d $NTHREADS_REGRID_NEMSIO -cc depth" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher -j 1 -n ${npe_remap:-$npe_fcst} -N $npe_node_fcst -d $NTHREADS_REMAP -cc depth" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher -j 1 -n ${npe_fv3:-$npe_efcs} -N $npe_node_efcs -d $NTHREADS_FV3 -cc depth" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher -j 1 -n $LEVS -N $npe_node_efcs -d $NTHREADS_REGRID_NEMSIO -cc depth" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher -j 1 -n ${npe_np:-$npe_post} -N $npe_node_post -d $NTHREADS_NP -cc depth" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher -j 1 -n $npe_dwn -N $npe_node_dwn -d $NTHREADS_DWN cfp" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher -j 1 -n $npe_ecen -N $npe_node_ecen -d $NTHREADS_ECEN -cc depth" - - export NTHREADS_CHGRES=${nth_chgres:-12} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher -j 1 -n $npe_ecen -N $npe_node_ecen -d $NTHREADS_CALCINC -cc depth" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher -j 1 -n $npe_epos -N $npe_node_epos -d $NTHREADS_EPOS -cc depth" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth" - -elif [ $step = "vrfy" ]; then - - export IOBUF_PARAMS="*:size=32M:count=4:verbose" - export APRUNTRACK="$launcher -j1 -n1 -N1 -d1 -cc depth" - -elif [ $step = "postsnd" ]; then - export IOBUF_PARAMS="sigf*:size=128M:count=20:prefetch=0:verbose,gfs_collectiv*:size=128M:count=2:prefetch=0:verbose,*.snd:size=128M:count=3:prefetch=0:verbose,*.sfc:size=32M:count=3:prefetch=0:verbose,bufr.*:size=8M:count=20:prefetch=0:verbose" - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher -j 1 -n $npe_postsnd -N $npe_node_postsnd -d $NTHREADS_POSTSND -cc depth" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher -j 1 -n $npe_postsndcfp -N $npe_node_postsndcfp -d $NTHREADS_POSTSNDCFP cfp" - -elif [ $step = "awips" ]; then - nth_max=$(($npe_node_max / $npe_node_awips)) - - export NTHREADS_AWIPS=${nth_awips:-2} - [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher -j 1 -n $npe_awips -N $npe_node_awips -d $NTHREADS_AWIPS -cc depth cfp" - -elif [ $step = "gempak" ]; then - nth_max=$(($npe_node_max / $npe_node_gempak)) - - export NTHREADS_GEMPAK=${nth_gempak:-3} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN_GEMPAKCFP="$launcher -j 1 -n $npe_gempak -N $npe_node_gempak -d $NTHREADS_GEMPAK cfp" - -fi diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env deleted file mode 100755 index 481fdb2a72..0000000000 --- a/env/WCOSS_DELL_P3.env +++ /dev/null @@ -1,200 +0,0 @@ -#!/bin/ksh -x - -if [ $# -ne 1 ]; then - - echo "Must specify an input argument to set runtime environment variables!" - echo "argument can be any one of the following:" - echo "anal fcst post vrfy" - echo "eobs eupd ecen efcs epos" - echo "postsnd awips gempak" - exit 1 - -fi - -step=$1 - -# WCOSS_DELL_P3 information -export npe_node_max=28 -export launcher="mpirun -n" - -# Due to ESMF issue, fv3gfs model must run with npe_node_max=24 -if [ $step = "fcst" -o $step = "efcs" ]; then - export npe_node_max=24 -fi - -# Configure MPI environment -#export MPI_BUFS_PER_PROC=2048 -#export MPI_BUFS_PER_HOST=2048 -#export MPI_GROUP_MAX=256 -#export MPI_MEMMAP_OFF=1 -export MPI_LABELIO=YES -export MP_STDOUTMODE="ORDERED" -export KMP_STACKSIZE=2048M -export KMP_AFFINITY=scatter - -# get binding information -#export I_MPI_DEBUG=4 - -if [ $step = "prep" -o $step = "prepbufr" ]; then - - nth_max=$(($npe_node_max / $npe_node_prep)) - - export POE="NO" - export BACK=${BACK:-"YES"} - export sys_tp="Dell-p3" - -elif [ $step = "anal" ]; then - - nth_max=$(($npe_node_max / $npe_node_anal)) - - export NTHREADS_GSI=${nth_anal:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_anal:-$PBS_NP}}" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher \$ncmd" - - export NTHREADS_CYCLE=${nth_cycle:-14} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - npe_cycle=${ntiles:-6} - export APRUN_CYCLE="$launcher $npe_cycle" - - export NTHREADS_GAUSFCANL=1 - npe_gausfcanl=${npe_gausfcanl:-1} - export APRUN_GAUSFCANL="$launcher $npe_gausfcanl" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "eobs" ]; then - - nth_max=$(($npe_node_max / $npe_node_eobs)) - - export NTHREADS_GSI=${nth_eobs:-$nth_max} - [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max - export APRUN_GSI="$launcher ${npe_gsi:-${npe_eobs:-$PBS_NP}}" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "eupd" ]; then - - nth_max=$(($npe_node_max / $npe_node_eupd)) - - export NTHREADS_ENKF=${nth_eupd:-$nth_max} - [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max - export APRUN_ENKF="$launcher ${npe_enkf:-${npe_eupd:-$PBS_NP}}" - - if [ ${USE_CFP:-"NO"} = "YES" ]; then - export APRUNCFP="$launcher \$ncmd cfp" - fi - -elif [ $step = "fcst" ]; then - - nth_max=$(($npe_node_max / $npe_node_fcst)) - - export NTHREADS_FV3=${nth_fv3:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_fcst:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - - export NTHREADS_REMAP=${nth_remap:-2} - [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max - export APRUN_REMAP="$launcher ${npe_remap:-${npe_fcst:-$PBS_NP}}" - -elif [ $step = "efcs" ]; then - - nth_max=$(($npe_node_max / $npe_node_efcs)) - - export NTHREADS_FV3=${nth_efcs:-$nth_max} - [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max - export cores_per_node=$npe_node_max - export APRUN_FV3="$launcher ${npe_fv3:-${npe_efcs:-$PBS_NP}}" - - export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1} - [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max - export APRUN_REGRID_NEMSIO="$launcher $LEVS" - -elif [ $step = "post" ]; then - - nth_max=$(($npe_node_max / $npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max - export APRUN_NP="$launcher ${npe_np:-${npe_post:-$PBS_NP}}" - - export NTHREADS_DWN=${nth_dwn:-1} - [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max - export APRUN_DWN="$launcher ${npe_dwn:-$PBS_NP} cfp" - -elif [ $step = "ecen" ]; then - - nth_max=$(($npe_node_max / $npe_node_ecen)) - - export NTHREADS_ECEN=${nth_ecen:-$nth_max} - [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max - export APRUN_ECEN="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CHGRES=${nth_chgres:-14} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${nth_calcinc:-1} - [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max - export APRUN_CALCINC="$launcher ${npe_ecen:-$PBS_NP}" - - export NTHREADS_CYCLE=${nth_cycle:-14} - [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max - export APRUN_CYCLE="$launcher $npe_ecen" - -elif [ $step = "epos" ]; then - - nth_max=$(($npe_node_max / $npe_node_epos)) - - export NTHREADS_EPOS=${nth_epos:-$nth_max} - [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max - export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}" - -elif [ $step = "fv3ic" ]; then - - export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max} - [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max - export APRUN_CHGRES="time" - -elif [ $step = "postsnd" ]; then - - nth_max=$(($npe_node_max / $npe_node_postsnd)) - - export NTHREADS_POSTSND=${nth_postsnd:-1} - [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max - export APRUN_POSTSND="$launcher $npe_postsnd" - - export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1} - [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max - export APRUN_POSTSNDCFP="$launcher $npe_postsndcfp cfp" - -elif [ $step = "awips" ]; then - - nth_max=$(($npe_node_max / $npe_node_awips)) - - export NTHREADS_AWIPS=${nth_awips:-2} - [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max - export APRUN_AWIPSCFP="$launcher ${npe_awips:-$PBS_NP} cfp" - -elif [ $step = "gempak" ]; then - - nth_max=$(($npe_node_max / $npe_node_gempak)) - - export NTHREADS_GEMPAK=${nth_gempak:-1} - [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max - export APRUN_GEMPAKCFP="$launcher \$ntasks cfp" - - -fi diff --git a/env/gfs.ver b/env/gfs.ver deleted file mode 100644 index efd4a1d9c3..0000000000 --- a/env/gfs.ver +++ /dev/null @@ -1,22 +0,0 @@ -export gfs_ver=v15.0.0 - -export crtm_ver=2.3.0 -export hwrf_ver=v11.0.0 -export g2tmpl_ver=1.4.0 - -export grib_util_ver=1.1.0 -export util_shared_ver=1.0.6 -export cfp_intel_sandybridge_ver=1.1.0 -export iobuf_ver=2.0.7 -export ESMF_intel_sandybridge_ver=3_1_0rp5 -export ESMF_intel_haswell_ver=3_1_0rp5 -export gempak_ver=7.3.0 -export old_gempak_ver=6.32.0 -export NCL_gnu_sandybridge_ver=6.3.0 -export ncarg_intel_sandybridge_ver=6.1.0 -export dumpjb_ver=4.0.0 - -## FOLLOWING are used by JGDAS_TROPC -export obsproc_dump_ver=v4.0.0 -export obsproc_shared_bufr_dumplist_ver=v1.5.0 - diff --git a/tests/baseline_configs/config.nco_CONUS_25km_GFDLgrid.sh b/tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh similarity index 100% rename from tests/baseline_configs/config.nco_CONUS_25km_GFDLgrid.sh rename to tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_25km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_25km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..c16cf54885 --- /dev/null +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200208" +DATE_LAST_CYCL="20200208" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_3km_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_3km_FV3GFS_FV3GFS.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh diff --git a/tests/baseline_configs/config.nco_RRFS_CONUS_3km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_CONUS_3km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..88d0074519 --- /dev/null +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,18 @@ +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200208" +DATE_LAST_CYCL="20200208" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.nco_RRFS_SUBCONUS_3km_HRRR_RAP.sh b/tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.nco_RRFS_SUBCONUS_3km_HRRR_RAP.sh rename to tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 9ba6347940..f1731ac10e 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -50,12 +50,14 @@ grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS grid_RRFS_SUBCONUS_3km_HRRR_RAP -nco_CONUS_25km_GFDLgrid -nco_RRFS_CONUS_25km_HRRR_RAP -nco_RRFS_CONUS_3km_FV3GFS_FV3GFS -nco_RRFS_CONUS_3km_HRRR_RAP -nco_RRFS_SUBCONUS_3km_HRRR_RAP nco_ensemble +nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional +nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR +nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR new_ESGgrid new_GFDLgrid new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 36bdff562f..4308486807 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -186,7 +186,7 @@ EXPT_SUBDIR="" # that will point to a subdirectory (having the name of the grid being # used) under this directory. This variable should be set to a null # string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN) +# workflow configuration file (EXPT_CONFIG_FN). # # STMP: # The beginning portion of the directory that will contain cycle-dependent diff --git a/ush/setup.sh b/ush/setup.sh index 8b08d32548..4200c6e735 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1017,8 +1017,54 @@ fi # #----------------------------------------------------------------------- # -GWD_HRRRsuite_DIR="${GWD_HRRRsuite_BASEDIR}/${PREDEF_GRID_NAME}" +GWD_HRRRsuite_DIR="" if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then +# +# If in NCO mode, make sure that GWD_HRRRsuite_BASEDIR is set equal to +# FIXLAM_NCO_BASEDIR +# + if [ "${RUN_ENVIR}" = "nco" ]; then + + if [ "${GWD_HRRRsuite_BASEDIR}" != "${FIXLAM_NCO_BASEDIR}" ]; then + + gwd_hrrrsuite_basedir_orig="${GWD_HRRRsuite_BASEDIR}" + GWD_HRRRsuite_BASEDIR="${FIXLAM_NCO_BASEDIR}" + + if [ ! -z "${gwd_hrrrsuite_basedir_orig}" ]; then + print_err_msg_exit " +When RUN_ENVIR is set to \"nco\", the workflow assumes that the base +directory (GWD_HRRRsuite_BASEDIR) under which the grid-specific +subdirectories containing the gravity wave drag-related orography +statistics files for the FV3_HRRR suite are located is the same as the +base directory (FIXLAM_NCO_BASEDIR) under which the other fixed files +are located. Currently, this is not the case: + GWD_HRRRsuite_BASEDIR = \"${gwd_hrrrsuite_basedir_orig}\" + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" +Resetting GWD_HRRRsuite_BASEDIR to FIXLAM_NCO_BASEDIR. Reset value is: + GWD_HRRRsuite_BASEDIR = \"${GWD_HRRRsuite_BASEDIR}\"" + fi + + fi + + fi +# +# Check that GWD_HRRRsuite_BASEDIR exists and is a directory. +# + if [ ! -d "${GWD_HRRRsuite_BASEDIR}" ]; then + print_err_msg_exit "\ +The base directory (GWD_HRRRsuite_BASEDIR) under which the grid-specific +subdirectories containing the gravity wave drag-related orography files +for the FV3_HRRR suite should be located does not exist (or is not a +directory): + GWD_HRRRsuite_BASEDIR = \"${GWD_HRRRsuite_BASEDIR}\"" + fi + GWD_HRRRsuite_DIR="${GWD_HRRRsuite_BASEDIR}/${PREDEF_GRID_NAME}" +# +# Ensure that PREDEF_GRID_NAME is not set to a null string. Currently, +# only predefined grids can be used with the FV3_HRRR suite because +# orography statistics files required by this suite are available only +# for (some of) the predefined grids. +# if [ -z "${PREDEF_GRID_NAME}" ]; then print_err_msg_exit "\ A predefined grid name (PREDEF_GRID_NAME) must be specified when using @@ -1026,6 +1072,11 @@ the FV3_HRRR physics suite: CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\"" else +# +# Ensure that the directory GWD_HRRRsuite_DIR in which the orography +# statistics files required by the FV3_HRRR suite are located actually +# exists. +# if [ ! -d "${GWD_HRRRsuite_DIR}" ]; then print_err_msg_exit "\ The directory (GWD_HRRRsuite_DIR) that should contain the gravity wave @@ -1038,6 +1089,7 @@ drag related orography files for the FV3_HRRR suite is empty: GWD_HRRRsuite_DIR = \"${GWD_HRRRsuite_DIR}\"" fi fi + fi # #----------------------------------------------------------------------- From 913458185cc61a9335936de5caa5948229ea1e28 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 11 Feb 2021 10:34:21 -0700 Subject: [PATCH 020/203] develop: changes required to process global NetCDF files. (#424) * Changes required to process global NetCDF files. * Add netcdf option to valid_param_vals.sh * Remove nemsio from variable name. * Update comments for fn_grib2, fn_atm, and fn_sfc variables. * Remove netcdf FV3GFS filename and path syntax from HPSS code, add comments to explain to user. --- scripts/exregional_make_ics.sh | 49 ++++++++++---- scripts/exregional_make_lbcs.sh | 25 ++++---- ush/get_extrn_mdl_file_dir_info.sh | 100 +++++++++++++++++++++++++++++ ush/valid_param_vals.sh | 4 +- 4 files changed, 149 insertions(+), 29 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index f335fa1635..3431b8b25d 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -192,13 +192,19 @@ esac # Name of the external model from which we are obtaining the fields # needed to generate the ICs. # -# fn_atm_nemsio: -# Name (not including path) of the nemsio file generated by the external -# model that contains the atmospheric fields. +# fn_atm: +# Name (not including path) of the nemsio or netcdf file generated by the +# external model that contains the atmospheric fields. Currently used for +# GSMGFS and FV3GFS external model data. # -# fn_sfc_nemsio: -# Name (not including path) of the nemsio file generated by the external -# model that contains the surface fields. +# fn_sfc: +# Name (not including path) of the nemsio or netcdf file generated by the +# external model that contains the surface fields. Currently used for +# GSMGFS and FV3GFS external model data. +# +# fn_grib2: +# Name (not including path) of the grib2 file generated by the external +# model. Currently used for NAM, RAP, and HRRR external model data. # # input_type: # The "type" of input being provided to chgres_cube. This contains a combi- @@ -287,8 +293,8 @@ esac # the field_table for CDATE=2017100700 but is a non-prognostic variable. external_model="" -fn_atm_nemsio="" -fn_sfc_nemsio="" +fn_atm="" +fn_sfc="" fn_grib2="" input_type="" tracers_input="\"\"" @@ -374,8 +380,9 @@ case "${EXTRN_MDL_NAME_ICS}" in "GSMGFS") external_model="GSMGFS" - fn_atm_nemsio="${EXTRN_MDL_FNS[0]}" - fn_sfc_nemsio="${EXTRN_MDL_FNS[1]}" + + fn_atm="${EXTRN_MDL_FNS[0]}" + fn_sfc="${EXTRN_MDL_FNS[1]}" input_type="gfs_gaussian_nemsio" # For spectral GFS Gaussian grid in nemsio format. convert_nst=False tracers_input="[\"spfh\",\"clwmr\",\"o3mr\"]" @@ -393,8 +400,9 @@ case "${EXTRN_MDL_NAME_ICS}" in external_model="FV3GFS" tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - fn_atm_nemsio="${EXTRN_MDL_FNS[0]}" - fn_sfc_nemsio="${EXTRN_MDL_FNS[1]}" + + fn_atm="${EXTRN_MDL_FNS[0]}" + fn_sfc="${EXTRN_MDL_FNS[1]}" input_type="gaussian_nemsio" # For FV3-GFS Gaussian grid in nemsio format. convert_nst=True elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then @@ -402,6 +410,19 @@ case "${EXTRN_MDL_NAME_ICS}" in fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" convert_nst=False + + elif [ "${FV3GFS_FILE_FMT_ICS}" = "netcdf" ]; then + + external_model="FV3GFS" + + tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" + tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" + + fn_atm="${EXTRN_MDL_FNS[0]}" + fn_sfc="${EXTRN_MDL_FNS[1]}" + input_type="gaussian_netcdf" # For FV3-GFS Gaussian grid in netcdf format. + convert_nst=True + fi vgtyp_from_climo=True sotyp_from_climo=True @@ -528,8 +549,8 @@ settings=" 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm_nemsio}, - 'sfc_files_input_grid': ${fn_sfc_nemsio}, + 'atm_files_input_grid': ${fn_atm}, + 'sfc_files_input_grid': ${fn_sfc}, 'grib2_file_input_grid': \"${fn_grib2}\", 'cycle_mon': $((10#${mm})), 'cycle_day': $((10#${dd})), diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 6a9fc7365e..c847d11a84 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -192,13 +192,14 @@ esac # Name of the external model from which we are obtaining the fields # needed to generate the LBCs. # -# fn_atm_nemsio: -# Name (not including path) of the nemsio file generated by the external -# model that contains the atmospheric fields. +# fn_atm: +# Name (not including path) of the nemsio or netcdf file generated by the +# external model that contains the atmospheric fields. Currently used for +# GSMGFS and FV3GFS external model data. # -# fn_sfc_nemsio: -# Name (not including path) of the nemsio file generated by the external -# model that contains the surface fields. +# fn_grib2: +# Name (not including path) of the grib2 file generated by the external +# model. Currently used for NAM, RAP, and HRRR external model data. # # input_type: # The "type" of input being provided to chgres_cube. This contains a combi- @@ -268,8 +269,7 @@ esac # the field_table for CDATE=2017100700 but is a non-prognostic variable. external_model="" -fn_atm_nemsio="" -fn_sfc_nemsio="" +fn_atm="" fn_grib2="" input_type="" tracers_input="\"\"" @@ -381,16 +381,16 @@ for (( i=0; i<${num_fhrs}; i++ )); do # Set external model output file name and file type/format. Note that # these are now inputs into chgres_cube. # - fn_atm_nemsio="" + fn_atm="" fn_grib2="" case "${EXTRN_MDL_NAME_LBCS}" in "GSMGFS") - fn_atm_nemsio="${EXTRN_MDL_FNS[$i]}" + fn_atm="${EXTRN_MDL_FNS[$i]}" ;; "FV3GFS") if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - fn_atm_nemsio="${EXTRN_MDL_FNS[$i]}" + fn_atm="${EXTRN_MDL_FNS[$i]}" elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then fn_grib2="${EXTRN_MDL_FNS[$i]}" fi @@ -461,8 +461,7 @@ settings=" 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm_nemsio}, - 'sfc_files_input_grid': ${fn_sfc_nemsio}, + 'atm_files_input_grid': ${fn_atm}, 'grib2_file_input_grid': \"${fn_grib2}\", 'cycle_mon': $((10#${mm})), 'cycle_day': $((10#${dd})), diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index b241dad80c..56c7cf54b8 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -375,6 +375,44 @@ fi # fns=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. fns_on_disk=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. + + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + +# +#This whole section needs to be set based on what is in HPSS. They are placeholders for now. +# + + fns=( "" "" ) + suffix="" + fns=( "" ) + +# Set names of external files if searching on disk. + if [ "${MACHINE}" = "JET" ]; then + prefix="" + else + prefix="" + fi + fns_on_disk=( "" ) + +# Set names of external files if searching in an archive file, e.g. from +# HPSS. + prefix="" + fns_in_arcv=( "" ) + + fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}")")" + fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}")")" + + print_info_msg " +Fetching of external model files from NOAA HPSS is not yet supported for +this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) +combination: + extrn_mdl_name = \"${extrn_mdl_name}\" + fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" +Setting fns_on_disk and fns_in_arcv to arrays containing empty elements: + fns_on_disk = ${fns_on_disk_str} + fns_in_arcv = ${fns_in_arcv_str} +If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the +workflow to look for the external model files in a user-staged directory." fi ;; @@ -471,6 +509,38 @@ and analysis or forecast (anl_or_fcst): fns_on_disk=( "${fcst_hhh[@]/#/$prefix}" ) fns_in_arcv=( "${fcst_hhh[@]/#/$prefix}" ) + + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + + fcst_hhh=( "" ) + suffix="" + fns=( "" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="" + else + prefix="" + fi + fns_on_disk=( "" ) + + prefix="" + fns_in_arcv=( "" ) + + fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}")")" + fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}")")" + + print_info_msg " +Fetching of external model files from NOAA HPSS is not yet supported for +this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) +combination: + extrn_mdl_name = \"${extrn_mdl_name}\" + fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" +Setting fns_on_disk and fns_in_arcv to arrays containing empty elements: + fns_on_disk = ${fns_on_disk_str} + fns_in_arcv = ${fns_in_arcv_str} +If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the +workflow to look for the external model files in a user-staged directory." + fi ;; @@ -816,6 +886,36 @@ has not been specified for this external model: arcv_fns="${arcv_fns}gfs_pgrb2" + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + + if [ "${anl_or_fcst}" = "ANL" ]; then + arcv_fns="" + elif [ "${anl_or_fcst}" = "FCST" ]; then + last_fhr_in_netcdfa="" + first_lbc_fhr="" + last_lbc_fhr="" + if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then + arcv_fns="" + elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then + arcv_fns="" + else + arcv_fns=( "${arcv_fns}" "${arcv_fns}" ) + fi + fi + + arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}")")" + + print_info_msg " +Fetching of external model files from NOAA HPSS is not yet supported for +this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) +combination: + extrn_mdl_name = \"${extrn_mdl_name}\" + fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" +Setting arcv_fns to an array containing empty elements: + arcv_fns = ${arcv_fns_str} +If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the +workflow to look for the external model files in a user-staged directory." + fi arcv_fmt="tar" diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index d05591b0fb..a67a0e357e 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -37,8 +37,8 @@ valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_USE_USER_STAGED_EXTRN_FILES=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2") -valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2") +valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2" "netcdf") +valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2" "netcdf") valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid") valid_vals_PREEXISTING_DIR_METHOD=("delete" "rename" "quit") valid_vals_GTYPE=("regional") From b4bc8b3134ef7c8159f928aae5c23b1797cbf74e Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 18 Feb 2021 13:42:43 -0500 Subject: [PATCH 021/203] Add 'module purge' to the launch script for orion (#438) Co-authored-by: Chan-Hoo Jeon --- ush/launch_FV3LAM_wflow.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 702dbdf936..4c53c63184 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -95,6 +95,7 @@ if [ "$MACHINE" = "CHEYENNE" ]; then module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ module load rocoto elif [ "$MACHINE" = "ORION" ]; then + module purge module load contrib rocoto elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then module purge From c014b01af564676e15ed70cb37a93990212c1d84 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 5 Mar 2021 13:49:15 -0500 Subject: [PATCH 022/203] Make WE2E tests work on the WCOSS Dell and Cray (#450) * Add options for wcoss to we2e tests * Make cron work on wcoss_dell_p3 * Modify the baseline list file --- ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 17 +++++++++++ ...s_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh} | 0 ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 17 +++++++++++ ...US_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} | 0 ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 17 +++++++++++ .../config.pregen_grid_orog_sfc_climo.sh | 2 +- tests/baselines_list.txt | 7 +++-- tests/run_experiments.sh | 30 +++++++++++++++---- ush/generate_FV3LAM_wflow.sh | 18 +++++++++-- ush/launch_FV3LAM_wflow.sh | 7 +++-- ush/set_extrn_mdl_params.sh | 30 +++++++++++++++++++ 11 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename tests/baseline_configs/{config.grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS.sh => config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh} (100%) create mode 100644 tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename tests/baseline_configs/{config.grid_RRFS_SUBCONUS_3km_HRRR_RAP.sh => config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh} (100%) create mode 100644 tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..5ff6cc8230 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh similarity index 100% rename from tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS.sh rename to tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta.sh diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..514ed92e57 --- /dev/null +++ b/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_HRRR_RAP.sh b/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 100% rename from tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_HRRR_RAP.sh rename to tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..35adb5d945 --- /dev/null +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -0,0 +1,17 @@ +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20190901" +DATE_LAST_CYCL="20190901" +CYCL_HRS=( "18" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" diff --git a/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh b/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh index ba31e1c4cd..4f4653c25a 100644 --- a/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh +++ b/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh @@ -4,7 +4,7 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" FCST_LEN_HRS="06" LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index f1731ac10e..5405fce017 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -45,16 +45,19 @@ grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta +grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_SUBCONUS_3km_FV3GFS_FV3GFS -grid_RRFS_SUBCONUS_3km_HRRR_RAP +grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta +grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 +grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR nco_ensemble nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional +nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 5bc3e11411..00ceb63ca1 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -500,7 +500,11 @@ VERBOSE=\"${VERBOSE}\"" # ${pregen_basedir}/${PREDEF_GRID_NAME}, and pregen_dir can then be used # to set GRID_DIR, OROG_DIR, and/or SFC_CLIMO_DIR below. - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + pregen_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + pregen_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "HERA" ]; then pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then pregen_basedir="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" @@ -562,7 +566,11 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + GWD_HRRRsuite_BASEDIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + GWD_HRRRsuite_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "HERA" ]; then GWD_HRRRsuite_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then GWD_HRRRsuite_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" @@ -642,7 +650,11 @@ envir=\"\${EXPT_SUBDIR}\"" # # Set FIXLAM_NCO_BASEDIR. # - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + FIXLAM_NCO_BASEDIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + FIXLAM_NCO_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "HERA" ]; then FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" @@ -675,7 +687,11 @@ FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + COMINgfs="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" + elif [ "$MACHINE" = "HERA" ]; then COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" elif [ "$MACHINE" = "JET" ]; then COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" @@ -724,7 +740,11 @@ PTMP=\"${PTMP}\"" # if [ ${USE_USER_STAGED_EXTRN_FILES} = "TRUE" ]; then - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + extrn_mdl_source_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + extrn_mdl_source_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "HERA" ]; then extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" elif [ "$MACHINE" = "JET" ]; then extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 1d1cba5279..32aff261cd 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -359,7 +359,11 @@ if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then print_info_msg " Copying contents of user cron table to backup file: crontab_backup_fp = \"${crontab_backup_fp}\"" - crontab -l > ${crontab_backup_fp} + if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + cp_vrfy "/u/$USER/cron/mycrontab" "${crontab_backup_fp}" + else + crontab -l > ${crontab_backup_fp} + fi # # Below, we use "grep" to determine whether the crontab line that the # variable CRONTAB_LINE contains is already present in the cron table. @@ -382,7 +386,11 @@ Copying contents of user cron table to backup file: # string in crontab_line_esc_astr (in which case it does something more # than the command portion of the string in crontab_line_esc_astr does). # - grep_output=$( crontab -l | grep "^${crontab_line_esc_astr}$" ) + if [ "$MACHINE" = "WCOSS_DELL_P3" ];then + grep_output=$( grep "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" ) + else + grep_output=$( crontab -l | grep "^${crontab_line_esc_astr}$" ) + fi exit_status=$? if [ "${exit_status}" -eq 0 ]; then @@ -399,7 +407,11 @@ Adding the following line to the cron table in order to automatically resubmit FV3-LAM workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" - ( crontab -l; echo "${CRONTAB_LINE}" ) | crontab - + if [ "$MACHINE" = "WCOSS_DELL_P3" ];then + echo "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" + else + ( crontab -l; echo "${CRONTAB_LINE}" ) | crontab - + fi fi diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 4c53c63184..8161ada542 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -402,8 +402,11 @@ launch script for this experiment: # ly find lines in the crontab that contain exactly the string in cron- # tab_line_esc_astr without any leading or trailing characters. # - ( crontab -l | grep -v "^${crontab_line_esc_astr}$" ) | crontab - - + if [ "$MACHINE" = "WCOSS_DELL_P3" ];then + grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" > tmpfile && mv tmpfile "/u/$USER/cron/mycrontab" + else + ( crontab -l | grep -v "^${crontab_line_esc_astr}$" ) | crontab - + fi fi # # Print the workflow completion message to the launch log file. diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 891c70e6d5..08a9cad113 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -111,6 +111,12 @@ else "RAP") case $MACHINE in + "WCOSS_CRAY") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/hps/nco/ops/com/rap/prod" + ;; + "WCOSS_DELL_P3") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/hps/nco/ops/com/rap/prod" + ;; "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; @@ -128,6 +134,12 @@ else "HRRR") case $MACHINE in + "WCOSS_CRAY") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; + "WCOSS_DELL_P3") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; @@ -145,6 +157,12 @@ else "NAM") case $MACHINE in + "WCOSS_CRAY") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/nam/prod" + ;; + "WCOSS_DELL_P3") + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/nam/prod" + ;; "HERA") EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; @@ -279,6 +297,12 @@ else "RAP") case $MACHINE in + "WCOSS_CRAY") + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/hps/nco/ops/com/rap/prod" + ;; + "WCOSS_DELL_P3") + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/hps/nco/ops/com/rap/prod" + ;; "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; @@ -296,6 +320,12 @@ else "HRRR") case $MACHINE in + "WCOSS_CRAY") + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; + "WCOSS_DELL_P3") + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; From 00578553840c28b9e22d37cd9e22d67118f1370f Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 11 Mar 2021 14:01:03 -0500 Subject: [PATCH 023/203] Fix the 'fdiag' issue on the develop branch (#454) Remove all fdiag entries from FV3.input.yml --- ush/templates/FV3.input.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index ea728485e3..c1f2154814 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -43,8 +43,6 @@ FV3_RRFS_v1alpha: imfshalcnv: -1 lsm: 2 lsoil_lsm: 4 - atmos_model_nml: - fdiag: 3 FV3_RRFS_v1beta: gfs_physics_nml: @@ -56,8 +54,6 @@ FV3_RRFS_v1beta: imfshalcnv: -1 lsm: 2 lsoil_lsm: 4 - atmos_model_nml: - fdiag: 3 FV3_HRRR: fv_core_nml: @@ -221,7 +217,6 @@ FV3_GFS_2017_gfdlmp_regional: FV3_GFS_v15p2: atmos_model_nml: ccpp_suite: FV3_GFS_v15 - fdiag: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 18, 21, 24] fms_nml: domains_stack_size: 1800200 fv_core_nml: &gfs_v15_fv_core @@ -312,7 +307,6 @@ FV3_GFS_v15p2: FV3_GFS_v16beta: atmos_model_nml: ccpp_suite: FV3_GFS_v16 - fdiag: 3 fhmax: 240 fhmaxhf: 0 fhout: 3 @@ -386,7 +380,6 @@ FV3_GFS_v16beta: FV3_CPT_v0: atmos_model_nml: ccpp_suite: FV3_CPT_v0 - fdiag: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 18, 21, 24] fms_nml: domains_stack_size: 1800200 fv_core_nml: From 21da612f39ed52ff96764691dca5c8bcd7f18481 Mon Sep 17 00:00:00 2001 From: Jeff-Duda <31049415+Jeff-Duda@users.noreply.github.com> Date: Mon, 15 Mar 2021 15:03:51 -0700 Subject: [PATCH 024/203] Sub-hourly post-processing enabled by some new config.sh entries. (#439) ## DESCRIPTION OF CHANGES: Modified the Jinja-formatted FV3LAM_wflow.xml template workflow to accommodate sub-hourly post-processing tasks that rely on sub-hourly FV3 output as a dependency. All changes are _additions_ to existing code and include the addition of a few keyword variables in the config.sh script. These new flags include... - SUB_HOURLY_POST: a logical flag indicating whether nor not sub-hourly post-processing is to be used - DT_SUBHOURLY_POST_MNTS: the increment in minutes to sub-divide the hour Additional post-processing tasks were added to FV3LAM_wflow.xml to account for the different FV3 output file names depending on whether sub-hourly FV3 output is used (the first FV3 output file has a different naming structure than the remaining output files). setup.sh was updated to check whether valid entries were used for these two variables and also check that DT_ATMOS divides evenly into DT_SUBHOURLY_POST_MNTS so that the FV3 output is consistent with the requested frequency of UPP output. config_defaults.sh and valid_param_vals.sh were also updated accordingly. ## TESTS CONDUCTED: Have run generate_FV3LAM_wflow.sh on a large variety of settings of SUB_HOURLY_POST and DT_SUBHOURLY_POST_MNTS. Note that setup.sh is configured such that DT_SUBHOURLY_POST_MNTS = 0 will cause SUB_HOURLY_POST to be ignored. I have successfully tested cases in which DT_ATMOS _does not_ divide evenly into DT_SUBHOURLY_POST_MNTS and when DT_SUBHOURLY_POST_MNTS is specified as anything other than a two-digit value (strings vs. open integers both work). The resulting workflows run successfully with rocotorun and output no error messages. ## ISSUE: Resolves issue #434 ## CONTRIBUTORS: @gsketefian. Contributions: 1) Fixed bug in setup.sh in the test that checks whether DT_SUBHOURLY_POST_MNTS is set to 0: should use the -eq operator instead of ==. 2) Fixed bug in the jinja XML template for rocoto (FV3LAM_wflow.xml) as follows: rearranged the post-processing tasks so that the post task is run for only the first minute of the last hour (e.g. if the forecast is 3 hours long, post is run for 3:00 but not for 3:15, 3:30, etc). 3) Ran the following 3 WE2E tests [note that tests (b) and (c) are not yet in the regional_workflow repo and will be included in a future PR]: a) **grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta**. This is without subhourly post-processing, i.e. SUB_HOURLY_POST is set to "FALSE". b) **subhourly_post**. This is with subhourly post-processing, i.e. SUB_HOURLY_POST set is set to "TRUE" (with DT_SUBHOURLY_POST_MNTS set to "12" minutes). c) **subhourly_post_ensemble_2mems**. This is with subhourly post-processing and with ensemble forecasts enabled, i.e. SUB_HOURLY_POST and DO_ENSEMBLES are both set to "TRUE" (with DT_SUBHOURLY_POST_MNTS is set to "12" minutes and NUM_ENS_MEMBERS set to "2" members). This test is run because the changes in the jinja XML template FV3LAM_wflow.xml needed to add subhourly post invovle code that executes ensemble forecasts. **All three tests were successful.** Note that this PR changes the names of the grib2 files that UPP generates such that they now always include the minutes -- regardless of whether SUB_HOURLY_POST is set to "TRUE" or "FALSE" (if set to "FALSE", the minutes are always "00"). For example, previously, the grib2 file for forecast hour 1 was named `rrfs.t00z.bgdawpf001.tm00.grib2`; henceforth, it will be named `rrfs.t00z.bgdawpf00100.tm00.grib2`. --- jobs/JREGIONAL_RUN_FCST | 1 + jobs/JREGIONAL_RUN_POST | 19 ++- scripts/exregional_run_fcst.sh | 5 +- scripts/exregional_run_post.sh | 39 ++++-- ush/config_defaults.sh | 12 +- ush/create_model_configure_file.sh | 20 ++- ush/generate_FV3LAM_wflow.sh | 6 + ush/setup.sh | 57 ++++++++ ush/templates/FV3LAM_wflow.xml | 202 ++++++++++++++++++++++++++++- ush/templates/model_configure | 4 +- ush/valid_param_vals.sh | 1 + 11 files changed, 342 insertions(+), 24 deletions(-) diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index a4806c765e..538b4a83c6 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -73,6 +73,7 @@ mkdir_vrfy -p ${run_dir}/RESTART # #----------------------------------------------------------------------- # + $SCRIPTSDIR/exregional_run_fcst.sh \ cdate="${CDATE}" \ cycle_dir="${CYCLE_DIR}" \ diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index e4e2122e46..f54667ad4b 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -88,7 +88,7 @@ else fi mkdir_vrfy -p "${postprd_dir}" -fhr_dir="${postprd_dir}/$fhr" +fhr_dir="${postprd_dir}/$fhr$fmn" check_for_preexist_dir_file "${fhr_dir}" "delete" mkdir_vrfy -p "${fhr_dir}" @@ -110,6 +110,19 @@ fi # #----------------------------------------------------------------------- # +# check if all necessary environmental variables are included +# +#----------------------------------------------------------------------- +# +if [ -n "${fmn}" ] && [ -z "${DT_ATMOS}" ]; then + print_err_msg_exit "\ +ERROR: Environmental variable \$fmn is present, so \$dt_atmos is also required +to be set (use same value as in FV3 namelist). This is needed so that +UPP will process the correct file for the first output timestep model fields." +fi +# +#----------------------------------------------------------------------- +# # Call the ex-script for this J-job and pass to it the necessary varia- # bles. # @@ -120,7 +133,9 @@ $SCRIPTSDIR/exregional_run_post.sh \ run_dir="${run_dir}" \ postprd_dir="${postprd_dir}" \ fhr_dir="${fhr_dir}" \ - fhr="${fhr}" || \ + fhr="${fhr}" \ + fmn="${fmn}" \ + dt_atmos="${DT_ATMOS}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index e712fb2b57..f59b0d3b3a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -451,7 +451,10 @@ fi create_model_configure_file \ cdate="$cdate" \ nthreads=${OMP_NUM_THREADS:-1} \ - run_dir="${run_dir}" || print_err_msg_exit "\ + run_dir="${run_dir}" \ + sub_hourly_post="${SUB_HOURLY_POST}" \ + dt_subhourly_post_mnts="${DT_SUBHOURLY_POST_MNTS}" \ + dt_atmos="${DT_ATMOS}" || print_err_msg_exit "\ Call to function to create a model configuration file for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index eaf79e12b3..1cddd61cc8 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -61,6 +61,8 @@ valid_args=( \ "postprd_dir" \ "fhr_dir" \ "fhr" \ +"fmn" \ +"dt_atmos" \ ) process_args valid_args "$@" # @@ -204,20 +206,32 @@ tmmark="tm00" # #----------------------------------------------------------------------- # -dyn_file="${run_dir}/dynf${fhr}.nc" -phy_file="${run_dir}/phyf${fhr}.nc" +mnts_secs_str="" +if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then + if [ ${fhr}${fmn} = "00000" ]; then + mnts_secs_str=":"`date --utc --date "${yyyymmdd} ${hh} UTC + ${dt_atmos} seconds" +%M:%S` + else + mnts_secs_str=":${fmn}:00" + fi +else + fmn="00" +fi + +dyn_file="${run_dir}/dynf${fhr}${mnts_secs_str}.nc" +phy_file="${run_dir}/phyf${fhr}${mnts_secs_str}.nc" +post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours +${fmn} minutes" "+%Y%m%d%H%M" ) -post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) post_yyyy=${post_time:0:4} post_mm=${post_time:4:2} post_dd=${post_time:6:2} post_hh=${post_time:8:2} +post_mn=${post_time:10:2} cat > itag <CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# - + @@ -403,19 +403,205 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> + {% if sub_hourly_post %} + + + + 000 + 00 + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + cyc@H + fhr#fhr# + fmn#fmin# + +{# +Note that because the forecast minutes and seconds corresponding to the +first model output time are not exactly zero, we use the jinja template +variable first_fv3_file_tstr instead of fhr and fmin to form the file +names in the dependencies. +#} + + + + + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf{{ first_fv3_file_tstr }}.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf{{ first_fv3_file_tstr }}.nc + + + + + + + + + + + + 000 + + + {% for min in range(delta_min, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + cyc@H + fhr#fhr# + fmn#fmin# + + + + + + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc + + + + + + + + + + {% endif %} + + {% if sub_hourly_post %} + + {% for h in range(1, fcst_len_hrs) %}{{ " %03d" % h }}{% endfor %} + + {% for min in range(0, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} + {% else %} + + {% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} + + 00 + {% endif %} + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" + {{ nnodes_run_post }}:ppn={{ ppn_run_post }} + {{ wtime_run_post }} + &NCORES_PER_NODE; + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + cyc@H + fhr#fhr# + fmn#fmin# + + + + + + {% if sub_hourly_post %} + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc + {% else %} + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#.nc + {% endif %} + + + + + + + + + - {% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} +{% if sub_hourly_post %} + + - + {{ "%03d" % fcst_len_hrs }} + 00 + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -424,13 +610,14 @@ MODULES_RUN_TASK_FP script. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H fhr#fhr# + fmn#fmin# - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#.nc - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc @@ -438,6 +625,7 @@ MODULES_RUN_TASK_FP script. +{% endif %} {% if do_ensemble %} diff --git a/ush/templates/model_configure b/ush/templates/model_configure index 33b0e527c9..9f4551181d 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -49,10 +49,10 @@ write_fsyncflag: .false. # Output frequency in time steps (positive values override "nfhout" and # "nfhout_hf"). # -nfhout: 1 +nfhout: {{ nfhout }} nfhmax_hf: 60 nfhout_hf: 1 -nsout: -1 +nsout: {{ nsout }} # # Coordinate system used by the output grid. # diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a67a0e357e..dce4752a8b 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -61,3 +61,4 @@ valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_COMPILER=("intel" "gnu") +valid_vals_SUB_HOURLY_POST=("TRUE" "true" "FALSE" "false" "YES" "yes" "NO" "no") From 846a1aa460b68fc3c4bca66bb00eb4665c481e9b Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Fri, 19 Mar 2021 10:49:25 -0600 Subject: [PATCH 025/203] Feature/add metplus (#219) * Fix to post flat file. * Create MET and METplus config files under ush/templates/parm * Added script to pull and reorg ccpa data. Added a script to run gridstat with METplus. Updated MET and METplus config files. * Added new jjob for running grid-stat vx. Updated setup.sh to include grid-stat vx. Updated run_gridstatvx script. * Fixed typo on script name from ksh to sh * Moved some hard coded items out from the script to the XML * Updates to get METplus to run with fewer hard-coded paths. * Updates to add grid-stat task to XML generation. * Bug fixes for adding grid-stat to XML generation * Updates to remove hard-coded paths in config files * Change log dir to put master_metplus log file with other logs under log/, rather than default logs/. * Updates to generate xml without hard-coded paths for MET * Add hera gridstat module file * Add METplus point-stat task for both sfc and upper air * Small tweaks to remove hard coded paths and add some flexibility * Updates for adding point-stat into auto-generated xml * Add in function to set point-stat task to FALSE * Final tweaks to get it to generate the xml correctly * Minor updates to run ensure 0,6,12,18 * Tweaks to var list for Point-Stat * Add METplus settings to config_defaults * Move quote for end of settings and fix extra comment. * Fix typos to populate templates correctly * Updated to include SCRIPTSDIR and other MET specific settings along with updates to FHR syntax * Update module loads on hera * Fixed comment for BOTH_VARn_THRESH to avoid syntax issues * Added files to run grid_stat for a variety of accumulation intervals, including 3, 6, and 24h * Added module load hpss * Remove module load informatino from these scripts * Updated the method of turning on/off vx tasks using jinja template if statement * Remove commented out lines of code. Fixed typo. Removed gen_wflow.out file. * Updated pull scripts to have file names dependent on date to pull from HPSS. Updated to export a few more local variables that METplus conf needed in scripts. Updated workflow to use service queue (for now) to for 1h grid_stat and point_stat run and default for 3+h accumulation grid_stat runs) * moved common_hera.conf to common.conf - no platform specific information included that needs to be handled. * Remove common_hera.conf * Add scripts to pull and process MRMS data from NOAA HPSS * Updates for REFC vx tasks * updates to obs pull scripts * Update for adding in reflectivity verification using MRMS analyses and updating name of model output to RRFS rather than HRRR * Updates to account for CCPA issues on HPSS - day off for 00-05 UTC directories * Verification mods to feature/add metplus (#1) * Remove unused/outdated code (#313) ## DESCRIPTION OF CHANGES: * In setup.sh and generate_FV3LAM_wflow.sh, remove temporary codes that fix bugs in the FV3_GFS_2017_gfdlmp_regional suite definition file because those bugs have been fixed (in the ufs-weather-model repo). * In setup.sh, remove block of code that is no longer necessary because chgres_cube can now initialize from external model data with either 4 or 9 soil levels, and run with LSMs of either 4 or 9 soil levels. * Remove modifications to LD_LIBRARY_PATH in exregional_run_fcst.sh. * For the make_ics and make_lbcs tasks, move the setting of APRUN and other machine-specific actions from the J-job to the ex-script in order to be consistent with the other workflow tasks. * Fix indentation and edit comments. * Remove unused file load_fv3gfs_modules.sh. ## TESTS CONDUCTED: Ran two WE2E tests on hera, new_ESGgrid and new_GFDLgrid: * new_ESGgrid uses the FV3_GFS_2017_gfdlmp_regional suite. The test was successful. * new_GFDLgrid uses the FV3_GFS_2017_gfdlmp suite. The test was successful. ## ISSUE (optional): This resolves issue #198. * Add and call a function that checks for use of Thompson microphysics parameterization in the SDF and if so, adjusts certain workflow arrays to contain the names and other associated values of the fixed files needed by this parameterization so that those files are automatically copied and/or linked to. (#319) ## DESCRIPTION OF CHANGES: Add and call a function that checks for use of Thompson microphysics parameterization in the suite definition file (SDF). If not, do nothing. If so, add to the appropriate workflow arrays the names and other associated values of the fixed files needed by this parameterization so that they are automatically copied and/or linked to instead of being regenerated from scratch in the run_fcst task. ## TESTS CONDUCTED: On hera, ran two WE2E tests, one in NCO mode (nco_RRFS_CONUS_25km_HRRRX_RAPX) and the other in community mode (suite_FV3_GSD_v0). These use suites FV3_GSD_SAR and FV3_GSD_v0, respectively, and both of these call Thompson microphysics. Both succeeded. ## ISSUE (optional): This PR resolves issue #297. * RRFS_v1beta SDF changes after reverting from GSL to GFS GWD suite (#322) (#327) ## DESCRIPTION OF CHANGES: Removed checks on the RRFS_v1beta SDF implemented for use with the GSL GWD suite (now uses the GFS GWD suite). No longer copies staged orography files necessary for the GSL GWD suite. ## TESTS CONDUCTED: Runs to completion on Hera. End-to-end runs DOT_OR_USCORE and suite_FV3_RRFS_v1beta succeeded on Cheyenne. Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> * Update FV3.input.nml for fhzero = 1.0 * Updated conf files for file name conventions. * Updated MET scripts and MRMS pull scripts. * Adjust RRFS_CONUS_... grids (#294) ## DESCRIPTION OF CHANGES: * Adjust RRFS_CONUS_25km, RRFS_CONUS_13km, and RRFS_CONUS_3km grid parameters so that: * All grids, including their 4-cell-wide halos, lie completely within the HRRRX domain. * All grids have dimensions nx and ny that factor "nicely", i.e. they don't have factors greather than 7. * The write-component grids corresponding to these three native grids cover as much of the native grids as possible without going outside of the native grid boundaries. The updated NCL scripts (see below) were used to generate the write-component grid parameters. * For the RRFS_CONUS_13km grid, reduce the time step (DT_ATMOS) from 180sec to 45sec. This is necessary to get a successful forecast with the GSD_SAR suite, and thus likely also the RRFS_v1beta suite. * Modify WE2E testing system as follows: * Add new tests with the RRFS_CONUS_25km, RRFS_CONUS_13km, and RRFS_CONUS_3km grids that use the GFS_v15p2 and RRFS_v1beta suites (which are now the ones officially supported in the first release of the short-range weather app) instead of the GFS_v16beta and GSD_SAR suites, respectively. * For clarity, rename the test configuration files that use the GFS_v16beta and GSD_SAR suites so they include the suite name. * Update list of WE2E tests (baselines_list.txt). * Update the NCL plotting scripts to be able to plot grids with the latest version of the workflow. ## TESTS CONDUCTED: On hera, ran tests with all three grids with the GFS_v15p2 and RRFS_v1beta suites (a total of 6 tests). All were successful. * Remove redundant model_configure.${CCPP_PHYS_SUITE} template files; use Jinja2 to create model_configure (#321) ## DESCRIPTION OF CHANGES: * Remove model_configure template files whose names depend on the physics suite, i.e. files with names of the form model_configure.${CCPP_PHYS_SUITE}. Only a single template file is needed because the contents of the model_configure file are not suite dependent. This leaves just one template file (named model_configure). * Change the function create_model_configure_file.sh and the template file model_configure so they use jinja2 instead of sed to replace placeholder values. * Absorb the contents of the write-component template files wrtcmp_lambert_conformal, wrtcmp_regional_latlon, and wrtcmp_rotated_latlon into the new jinja2-compliant model_configure file. We can do this because Jinja2 allows use of if-statements in the template file. * In the new model_configure jinja2 template file, include comments to explain the various write-component parameters. ## TESTS CONDUCTED: On Hera, ran the two WE2E tests new_ESGgrid and new_GFDLgrid. The first uses a "lambert_conformal" type of write-component grid, and the second uses a "rotated_latlon" type of write-component grid. (The write-component also allows "regional_latlon" type grids, which is just the usual earth-relative latlon coordinate system, but we do not have any cases that use that.) Both tests succeeded. ## ISSUE (optional): This PR resolves issue #281. * Add Thompson ice- and water-friendly aerosol climo file support (#332) * Add if statement in set_thompson_mp_fix_files.sh to source Thompson climo file when using a combination of a Thompson-based SDF and non-RAP/HRRR external model data * Modify if statement based on external models for Thompson climo file * Remove workflow variable EMC_GRID_NAME (#333) ## DESCRIPTION OF CHANGES: * Remove the workflow variable EMC_GRID_NAME. Henceforth, PREDEF_GRID_NAME is the only variable that can be used to set the name of the predefined grid to use. * Make appropriate change of variable name (EMC_GRID_NAME --> PREDEF_GRID_NAME) in the WE2E test configuration files. * Change anywhere the "conus" and "conus_c96" grids are specified to "EMC_CONUS_3km" and "EMC_CONUS_coarse", respectively. * Rename WE2E test configuration files with names containing the strings "conus" and "conus_c96" by replacing these strings with "EMC_CONUS_3km" and "EMC_CONUS_coarse", respectively. * Update the list of WE2E test names (tests/baselines_list.txt). * Bug fixes not directly related to grids: * In config.nco.sh, remove settings of QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST since these are now set automatically (due to another PR). * In the template file FV3LAM_wflow.xml, add the ensemble member name after RUN_FCST_TN in the dependency of the run_post metatask. ## TESTS CONDUCTED: Since this change only affects runs in NCO mode, the following NCO-mode WE2E tests were rerun on hera, all successfully: ``` nco_EMC_CONUS_3km SUCCESS nco_EMC_CONUS_coarse SUCCESS nco_EMC_CONUS_coarse__suite_FV3_GFS_2017_gfdlmp SUCCESS nco_RRFS_CONUS_25km_HRRRX_RAPX SUCCESS nco_RRFS_CONUS_3km_FV3GFS_FV3GFS SUCCESS nco_RRFS_CONUS_3km_HRRRX_RAPX SUCCESS nco_ensemble SUCCESS ``` * Port workflow to Orion (#309) ## DESCRIPTION OF CHANGES: * Add stanzas for Orion where necessary. * Add new module files for Orion. * On Orion, both the slurm partition and the slurm QOS need to be specified in the rocoto XML in order to be able to have wall times longer than 30 mins (the partition needs to be specified because it is by default "debug", which has a limit of 30 mins). Thus, introduce modifications to more easily specify slurm partitions: * Remove the workflow variables QUEUE_DEFAULT_TAG, QUEUE_HPSS_TAG, and QUEUE_FCST_TAG that are currently used to determine whether QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST specify the names of queue/QOS's or slurm partitions. * Add the workflow variables PARTITION_DEFAULT_TAG, PARTITION_HPSS_TAG, and PARTITION_FCST_TAG. These will be used to specify slurm partitions only, and the variables QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST will be used to specify queues/QOS's only. IMPORTANT NOTE: On Orion, in order to load the regional_workflow environment needed for generating an experiment, the user must first issue the following commands: ``` module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 conda activate regional_workflow ``` ## TESTS CONDUCTED: Ran 11 WE2E tests on Orion, Hera, and Cheyenne. Results on Orion: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km FAILURE - In the run_fcst task. * Error message: !!! (1) Error in subr radiation_aerosols: unrealistic surface pressure = 1 NaN new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` Results on Hera: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km SUCCESS new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` Results on Cheyenne: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km FAILURE - In run_fcst task. * Error message: !!! (1) Error in subr radiation_aerosols: unrealistic surface pressure = 1 NaN new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` All succeed except GSD_HRRR_AK_50km on Orion and Cheyenne. It is not clear why grid_GSD_HRRR_AK_50km fails on Orion and Cheyenne but not Hera. Seems to point to a bug in the forecast model. These two failures are not so important since this grid will soon be deprecated. Also tested successfully on Jet by @JeffBeck-NOAA and on Odin and Stampede by @ywangwof. ## ISSUE: This resolves Issue #152. ## CONTRIBUTORS: @JeffBeck-NOAA @ywangwof @christinaholtNOAA * Removed comments from exregional_get_mrms_files.sh and removed fhzero from FV3.input.yml * Update FV3.input.nml for fhzero = 1.0 * Updated conf files for file name conventions. * Updated MET scripts and MRMS pull scripts. * Removed comments from exregional_get_mrms_files.sh and removed fhzero from FV3.input.yml Co-authored-by: gsketefian <31046882+gsketefian@users.noreply.github.com> Co-authored-by: Michael Kavulich Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Co-authored-by: Jamie Wolff * Change cov_thresh for REFL to be a true max in nbrhood as SPC does. * Separated Pull Data Scripts from Run Vx Scripts: Feature/add_metplus (#2) * Job script for get_obs_ccpa * Jobs script for get_obs_mrms * Jobs script for get_obs_ndas * Added external variables necessary to get_ccpa script * Updated workflow template with separate get obs tasks * Separated pull scripts from run scripts * Added necessary defaults/values for defining pull tasks * Added module files, default config.sh options, and changed dependencies for vx tasks * Changed name of new workflow to FV3LAM_wflow.xml * Added task get_obs_tn, removed config.sh, updated config_defaults and config.community.sh * Adjusted the community and default config files based on comments * Updated FV3LAM workflow * Fixed discrepancies in config.community.sh * Fixed discrepancies in config_defaults.sh * Fixed discrepancies in config_defaults.sh round 2 * Fixed discrepancies in config_defaults.sh round 3 * Fixed discrepancies in config_defaults.sh round 4 * Fixed discrepancies in config.community.sh round 2 * Fixed discrepancies in config.community.sh round 3 * Fixed discrepancies in generate_FV3LAM_wflow.sh * Fixed discrepancies in generate_FV3LAM_wflow.sh round 2 * Fixed discrepancies in generate_FV3LAM_wflow.sh round 3 * Updated FV3LAM_wflow template * Fixed Vx Task Dependencies in Workflow: Feature/add metplus (#3) * Job script for get_obs_ccpa * Jobs script for get_obs_mrms * Jobs script for get_obs_ndas * Added external variables necessary to get_ccpa script * Updated workflow template with separate get obs tasks * Separated pull scripts from run scripts * Added necessary defaults/values for defining pull tasks * Added module files, default config.sh options, and changed dependencies for vx tasks * Changed name of new workflow to FV3LAM_wflow.xml * Added task get_obs_tn, removed config.sh, updated config_defaults and config.community.sh * Adjusted the community and default config files based on comments * Updated FV3LAM workflow * Fixed discrepancies in config.community.sh * Fixed discrepancies in config_defaults.sh * Fixed discrepancies in config_defaults.sh round 2 * Fixed discrepancies in config_defaults.sh round 3 * Fixed discrepancies in config_defaults.sh round 4 * Fixed discrepancies in config.community.sh round 2 * Fixed discrepancies in config.community.sh round 3 * Fixed discrepancies in generate_FV3LAM_wflow.sh * Fixed discrepancies in generate_FV3LAM_wflow.sh round 2 * Fixed discrepancies in generate_FV3LAM_wflow.sh round 3 * Updated FV3LAM_wflow template * Fixed the dependencies of the vx tasks * Manual merge with develop that didn't seem to work before. Trying to get feature branch updated so it will run again! * Add local module files * Add environment variable for SCRIPTSDIR * Remove echo statement * Remove old module files * Update to config_default for walltime for ndas pull. Update to metplus parm for obs file template. Update to FV3LAM xml to not include 00 hour for verification * Update template to remove full path * Verification channges for obs. (#4) * Verification channges for obs. * Update config_defaults.sh for vx description * Update config_defaults.sh to remove extraneous MET info. Co-authored-by: Michelle Harrold * Pull in updates from develop that were not merging properly. Small change to config.community to turn off vx tasks by default. * Did manual merge of these files because it was not handled properly automatically * Adding additional variables to METplus for regional workflow (#5) * Updates to address comments in PR review * Changed name of get_obs to remove _tn * Missed removal of on get_obs Co-authored-by: michelleharrold Co-authored-by: gsketefian <31046882+gsketefian@users.noreply.github.com> Co-authored-by: Michael Kavulich Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Co-authored-by: Lindsay Co-authored-by: Michelle Harrold Co-authored-by: PerryShafran-NOAA <62255233+PerryShafran-NOAA@users.noreply.github.com> --- jobs/JREGIONAL_GET_OBS_CCPA | 94 +++++ jobs/JREGIONAL_GET_OBS_MRMS | 90 +++++ jobs/JREGIONAL_GET_OBS_NDAS | 90 +++++ jobs/JREGIONAL_RUN_VX_GRIDSTAT | 128 +++++++ jobs/JREGIONAL_RUN_VX_POINTSTAT | 126 +++++++ modulefiles/tasks/hera/get_obs.local | 9 + modulefiles/tasks/hera/run_vx.local | 6 + scripts/exregional_get_ccpa_files.sh | 218 +++++++++++ scripts/exregional_get_mrms_files.sh | 131 +++++++ scripts/exregional_get_ndas_files.sh | 145 ++++++++ scripts/exregional_run_gridstatvx.sh | 177 +++++++++ scripts/exregional_run_pointstatvx.sh | 161 ++++++++ scripts/mrms_pull_topofhour.py | 103 ++++++ ush/config.community.sh | 18 +- ush/config_defaults.sh | 153 +++++++- ush/generate_FV3LAM_wflow.sh | 57 +++ ush/setup.sh | 89 +++++ ush/templates/FV3.input.yml | 2 - ush/templates/FV3LAM_wflow.xml | 345 +++++++++++++++++- ush/templates/parm/met/GridStatConfig_APCP | 178 +++++++++ ush/templates/parm/met/GridStatConfig_REFL | 219 +++++++++++ ush/templates/parm/met/PB2NCConfig_conus_sfc | 161 ++++++++ ush/templates/parm/met/PB2NCConfig_upper_air | 157 ++++++++ .../parm/met/PointStatConfig_conus_sfc | 259 +++++++++++++ .../parm/met/PointStatConfig_upper_air | 262 +++++++++++++ ush/templates/parm/metplus/APCP_01h.conf | 91 +++++ ush/templates/parm/metplus/APCP_03h.conf | 121 ++++++ .../parm/metplus/APCP_03h.conf_3hCCPA | 108 ++++++ ush/templates/parm/metplus/APCP_06h.conf | 121 ++++++ .../parm/metplus/APCP_06h.conf_6hCCPA | 108 ++++++ ush/templates/parm/metplus/APCP_24h.conf | 121 ++++++ .../parm/metplus/PointStat_conus_sfc.conf | 172 +++++++++ .../parm/metplus/PointStat_upper_air.conf | 157 ++++++++ ush/templates/parm/metplus/REFC.conf | 94 +++++ ush/templates/parm/metplus/common.conf | 22 ++ ush/valid_param_vals.sh | 2 + 36 files changed, 4480 insertions(+), 15 deletions(-) create mode 100755 jobs/JREGIONAL_GET_OBS_CCPA create mode 100755 jobs/JREGIONAL_GET_OBS_MRMS create mode 100755 jobs/JREGIONAL_GET_OBS_NDAS create mode 100755 jobs/JREGIONAL_RUN_VX_GRIDSTAT create mode 100755 jobs/JREGIONAL_RUN_VX_POINTSTAT create mode 100644 modulefiles/tasks/hera/get_obs.local create mode 100644 modulefiles/tasks/hera/run_vx.local create mode 100755 scripts/exregional_get_ccpa_files.sh create mode 100755 scripts/exregional_get_mrms_files.sh create mode 100755 scripts/exregional_get_ndas_files.sh create mode 100755 scripts/exregional_run_gridstatvx.sh create mode 100755 scripts/exregional_run_pointstatvx.sh create mode 100644 scripts/mrms_pull_topofhour.py create mode 100755 ush/templates/parm/met/GridStatConfig_APCP create mode 100644 ush/templates/parm/met/GridStatConfig_REFL create mode 100644 ush/templates/parm/met/PB2NCConfig_conus_sfc create mode 100644 ush/templates/parm/met/PB2NCConfig_upper_air create mode 100644 ush/templates/parm/met/PointStatConfig_conus_sfc create mode 100644 ush/templates/parm/met/PointStatConfig_upper_air create mode 100644 ush/templates/parm/metplus/APCP_01h.conf create mode 100644 ush/templates/parm/metplus/APCP_03h.conf create mode 100644 ush/templates/parm/metplus/APCP_03h.conf_3hCCPA create mode 100644 ush/templates/parm/metplus/APCP_06h.conf create mode 100644 ush/templates/parm/metplus/APCP_06h.conf_6hCCPA create mode 100644 ush/templates/parm/metplus/APCP_24h.conf create mode 100644 ush/templates/parm/metplus/PointStat_conus_sfc.conf create mode 100644 ush/templates/parm/metplus/PointStat_upper_air.conf create mode 100644 ush/templates/parm/metplus/REFC.conf create mode 100644 ush/templates/parm/metplus/common.conf diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA new file mode 100755 index 0000000000..6f541ccd25 --- /dev/null +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -0,0 +1,94 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script pulls CCPA observation data for comparison to the model for +# the requested accumulations. Supported accumulations: 01h, 03h, and 06h. +# NOTE: Accumulation is currently hardcoded to 01h. +# The verification uses MET/pcp-combine to sum 01h files into +# desired accumulations. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that pulls CCPA observation data +for verification purposes. +========================================================================" + +# +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_get_ccpa_files.sh || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS new file mode 100755 index 0000000000..3524532c61 --- /dev/null +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -0,0 +1,90 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script pulls MRMS observation data for comparison to the model. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that pulls MRMS observation data +for verification purposes. +========================================================================" + +# +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_get_mrms_files.sh || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS new file mode 100755 index 0000000000..32c1d819e7 --- /dev/null +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -0,0 +1,90 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script pulls NDAS observation data for comparison to the model. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that pulls NDAS observation data +for verification purposes. +========================================================================" + +# +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_get_ndas_files.sh || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT new file mode 100755 index 0000000000..ca29e2d931 --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -0,0 +1,128 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script runs METplus for grid-stat on the UPP output files by +# initialization time for all forecast hours. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for grid-stat +by initialization time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# If it doesn't already exist, create the directory (vx_dir) in +# which to store grid-stat output. +# +# Note that there may be a preexisting version of vx_dir from previous +# runs of this script (e.g. from the workflow task that runs this script +# failing and then being called again). Thus, we first make sure +# preexisting versions are deleted. +# +#----------------------------------------------------------------------- +# + +if [ "${RUN_ENVIR}" = "nco" ]; then + COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" + postprd_dir="$COMOUT" +else + postprd_dir="${CYCLE_DIR}/postprd" +fi +mkdir_vrfy -p "${postprd_dir}" + +if [ "${RUN_ENVIR}" = "nco" ]; then + COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" + vx_dir="$COMOUT" +else + vx_dir="${CYCLE_DIR}/metprd" +fi +mkdir_vrfy -p "${vx_dir}" + +gridstat_dir="${vx_dir}/grid_stat" +#check_for_preexist_dir "${gridstat_dir}" "delete" +mkdir_vrfy -p "${gridstat_dir}" + +cd_vrfy "${gridstat_dir}" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_gridstatvx.sh \ + cycle_dir="${CYCLE_DIR}" \ + postprd_dir="${postprd_dir}" \ + vx_dir="${vx_dir}" \ + gridstat_dir="${gridstat_dir}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT new file mode 100755 index 0000000000..f5c35253a2 --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -0,0 +1,126 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for point-stat +by initialitation time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# If it doesn't already exist, create the directory (vx_dir) in +# which to store point-stat output. +# +# Note that there may be a preexisting version of vx_dir from previous +# runs of this script (e.g. from the workflow task that runs this script +# failing and then being called again). Thus, we first make sure +# preexisting versions are deleted. +# +#----------------------------------------------------------------------- +# + +if [ "${RUN_ENVIR}" = "nco" ]; then + COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" + postprd_dir="$COMOUT" +else + postprd_dir="${CYCLE_DIR}/postprd" +fi +mkdir_vrfy -p "${postprd_dir}" + +if [ "${RUN_ENVIR}" = "nco" ]; then + COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" + vx_dir="$COMOUT" +else + vx_dir="${CYCLE_DIR}/metprd" +fi +mkdir_vrfy -p "${vx_dir}" + +pointstat_dir="${vx_dir}/point_stat" +#check_for_preexist_dir "${pointstat_dir}" "delete" +mkdir_vrfy -p "${pointstat_dir}" + +cd_vrfy "${pointstat_dir}" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_pointstatvx.sh \ + cycle_dir="${CYCLE_DIR}" \ + postprd_dir="${postprd_dir}" \ + vx_dir="${vx_dir}" \ + pointstat_dir="${pointstat_dir}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/modulefiles/tasks/hera/get_obs.local b/modulefiles/tasks/hera/get_obs.local new file mode 100644 index 0000000000..47c1c6f31d --- /dev/null +++ b/modulefiles/tasks/hera/get_obs.local @@ -0,0 +1,9 @@ +#%Module##################################################### +## Module file for get_obs_ccpa, get_obs_mrms, and +## get_obs_ndas task. +############################################################# + +module purge + +module load hpss + diff --git a/modulefiles/tasks/hera/run_vx.local b/modulefiles/tasks/hera/run_vx.local new file mode 100644 index 0000000000..99c35f92e0 --- /dev/null +++ b/modulefiles/tasks/hera/run_vx.local @@ -0,0 +1,6 @@ +#%Module +module use -a /contrib/anaconda/modulefiles +module load anaconda/latest +module use -a /contrib/met/modulefiles/ +module load met/9.0.2 + diff --git a/scripts/exregional_get_ccpa_files.sh b/scripts/exregional_get_ccpa_files.sh new file mode 100755 index 0000000000..2354938f1a --- /dev/null +++ b/scripts/exregional_get_ccpa_files.sh @@ -0,0 +1,218 @@ +#!/bin/sh + +# This script reorganizes the CCPA data into a more intuitive structure: +# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. +# Supported accumulations: 01h, 03h, and 06h. NOTE: Accumulation is currently hardcoded to 01h. +# The verification uses MET/pcp-combine to sum 01h files into desired accumulations. + +# Top-level CCPA directory +ccpa_dir=${OBS_DIR}/.. +if [[ ! -d "$ccpa_dir" ]]; then + mkdir -p $ccpa_dir +fi + +# CCPA data from HPSS +ccpa_raw=$ccpa_dir/raw +if [[ ! -d "$ccpa_raw" ]]; then + mkdir -p $ccpa_raw +fi + +# Reorganized CCPA location +ccpa_proc=$ccpa_dir/proc +if [[ ! -d "$ccpa_proc" ]]; then + mkdir -p $ccpa_proc +fi + +# Accumulation is for accumulation of CCPA data to pull (hardcoded to 01h, see note above.) +#accum=${ACCUM} +accum=01 + +# Initialization +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh + +init=${CDATE}${hh} + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` + +# Forecast length +fcst_length=${fhr_last} + +current_fcst=$accum +while [[ ${current_fcst} -le ${fcst_length} ]]; do + # Calculate valid date info + fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds + yyyy=`echo ${init} | cut -c1-4` # year (YYYY) of initialization time + mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time + dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time + hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time + init_ut=`date -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time + vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time + vdate=`date -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time + vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time + vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time + vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) + + vhh_noZero=$(expr ${vhh} + 0) + + # Calculate valid date - 1 day + vdate_ut_m1=`expr ${vdate_ut} - 86400` + vdate_m1=`date -ud '1970-01-01 UTC '${vdate_ut_m1}' seconds' +%Y%m%d%H` + vyyyymmdd_m1=`echo ${vdate_m1} | cut -c1-8` + vyyyy_m1=`echo ${vdate_m1} | cut -c1-4` + vmm_m1=`echo ${vdate_m1} | cut -c5-6` + vdd_m1=`echo ${vdate_m1} | cut -c7-8` + vhh_m1=`echo ${vdate_m1} | cut -c9-10` + + # Calculate valid date + 1 day + vdate_ut_p1=`expr ${vdate_ut} + 86400` + vdate_p1=`date -ud '1970-01-01 UTC '${vdate_ut_p1}' seconds' +%Y%m%d%H` + vyyyymmdd_p1=`echo ${vdate_p1} | cut -c1-8` + vyyyy_p1=`echo ${vdate_p1} | cut -c1-4` + vmm_p1=`echo ${vdate_p1} | cut -c5-6` + vdd_p1=`echo ${vdate_p1} | cut -c7-8` + vhh_p1=`echo ${vdate_p1} | cut -c9-10` + + # Create necessary raw and prop directories + if [[ ! -d "$ccpa_raw/${vyyyymmdd}" ]]; then + mkdir -p $ccpa_raw/${vyyyymmdd} + fi + + if [[ ! -d "$ccpa_raw/${vyyyymmdd_m1}" ]]; then + mkdir -p $ccpa_raw/${vyyyymmdd_m1} + fi + + if [[ ! -d "$ccpa_raw/${vyyyymmdd_p1}" ]]; then + mkdir -p $ccpa_raw/${vyyyymmdd_p1} + fi + + if [[ ! -d "$ccpa_proc/${vyyyymmdd}" ]]; then + mkdir -p $ccpa_proc/${vyyyymmdd} + fi + + # Name of CCPA tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. + if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20190812 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" + fi + + if [[ ${vyyyymmdd_m1} -ge 20190101 && ${vyyyymmdd_m1} -lt 20190812 ]]; then + TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com2_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" + fi + + if [[ ${vyyyymmdd_p1} -ge 20190101 && ${vyyyymmdd_p1} -lt 20190812 ]]; then + TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com2_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" + fi + + if [[ ${vyyyymmdd} -ge 20190812 && ${vyyyymmdd} -le 20200217 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" + fi + + if [[ ${vyyyymmdd_m1} -ge 20190812 && ${vyyyymmdd_m1} -le 20200217 ]]; then + TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" + fi + + if [[ ${vyyyymmdd_p1} -ge 20190812 && ${vyyyymmdd_p1} -le 20200217 ]]; then + TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" + fi + + if [[ ${vyyyymmdd} -gt 20200217 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" + fi + + if [[ ${vyyyymmdd_m1} -gt 20200217 ]]; then + TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" + fi + + if [[ ${vyyyymmdd_p1} -gt 20200217 ]]; then + TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" + fi + + # Check if file exists on disk; if not, pull it. + ccpa_file="$ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" + echo "CCPA FILE:${ccpa_file}" + if [[ ! -f "${ccpa_file}" ]]; then + if [[ ${accum} == "01" ]]; then + # Check if valid hour is 00 + if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then + cd $ccpa_raw/${vyyyymmdd_p1} + # Pull CCPA data from HPSS + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + else + cd $ccpa_raw/${vyyyymmdd} + # Pull CCPA data from HPSS + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + fi + + # One hour CCPA files have incorrect metadeta in the files under the "00" directory. After data is pulled, reorganize into correct valid yyyymmdd structure. + if [[ ${vhh_noZero} -ge 1 && ${vhh_noZero} -le 6 ]]; then + cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 7 && ${vhh_noZero} -le 12 ]]; then + cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 13 && ${vhh_noZero} -le 18 ]]; then + cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + elif [[ ${vhh_noZero} -eq 0 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + fi + + elif [[ ${accum} == "03" ]]; then + # Check if valid hour is 21 + if [[ ${vhh_noZero} -ne 21 ]]; then + cd $ccpa_raw/${vyyyymmdd} + # Pull CCPA data from HPSS + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + elif [[ ${vhh_noZero} -eq 21 ]]; then + cd $ccpa_raw/${vyyyymmdd_p1} + # Pull CCPA data from HPSS + TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + fi + + if [[ ${vhh_noZero} -eq 0 ]]; then + cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 3 || ${vhh_noZero} -eq 6 ]]; then + cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 9 || ${vhh_noZero} -eq 12 ]]; then + cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 15 || ${vhh_noZero} -eq 18 ]]; then + cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 21 ]]; then + cp $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi + + elif [[ ${accum} == "06" ]]; then + cd $ccpa_raw/${vyyyymmdd} + # Pull CCPA data from HPSS + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` + + if [[ ${vhh_noZero} -eq 0 ]]; then + cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 6 ]]; then + cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 12 ]]; then + cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -eq 18 ]]; then + cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi + fi + fi + + # Increment to next forecast hour + current_fcst=$((${current_fcst} + ${accum})) + echo "Current fcst hr=${current_fcst}" + +done + diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh new file mode 100755 index 0000000000..01d95043d7 --- /dev/null +++ b/scripts/exregional_get_mrms_files.sh @@ -0,0 +1,131 @@ +#!/bin/sh + +# This script pulls MRMS data from the NOAA HPSS +# Top-level MRMS directory +mrms_dir=${OBS_DIR}/.. +if [[ ! -d "$mrms_dir" ]]; then + mkdir -p $mrms_dir +fi + +# MRMS data from HPSS +mrms_raw=$mrms_dir/raw +if [[ ! -d "$mrms_raw" ]]; then + mkdir -p $mrms_raw +fi + +# Reorganized MRMS location +mrms_proc=$mrms_dir/proc +if [[ ! -d "$mrms_proc" ]]; then + mkdir -p $mrms_proc +fi + +# Initialization +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh + +start_valid=${CDATE}${hh} + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` + +# Forecast length +fcst_length=${fhr_last} + +s_yyyy=`echo ${start_valid} | cut -c1-4` # year (YYYY) of start time +s_mm=`echo ${start_valid} | cut -c5-6` # month (MM) of start time +s_dd=`echo ${start_valid} | cut -c7-8` # day (DD) of start time +s_hh=`echo ${start_valid} | cut -c9-10` # hour (HH) of start time +start_valid_ut=`date -ud ''${s_yyyy}-${s_mm}-${s_dd}' UTC '${s_hh}':00:00' +%s` # convert start time to universal time + +end_fcst_sec=`expr ${fcst_length} \* 3600` # convert last forecast lead hour to seconds +end_valid_ut=`expr ${start_valid_ut} + ${end_fcst_sec}` # calculate current forecast time in universal time + +cur_ut=${start_valid_ut} +current_fcst=0 +fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds + +while [[ ${cur_ut} -le ${end_valid_ut} ]]; do + cur_time=`date -ud '1970-01-01 UTC '${cur_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + echo "cur_time=${cur_time}" + + # Calculate valid date info + vyyyy=`echo ${cur_time} | cut -c1-4` # year (YYYY) of time + vmm=`echo ${cur_time} | cut -c5-6` # month (MM) of time + vdd=`echo ${cur_time} | cut -c7-8` # day (DD) of time + vhh=`echo ${cur_time} | cut -c9-10` # hour (HH) of time + vyyyymmdd=`echo ${cur_time} | cut -c1-8` # YYYYMMDD of time + vinit_ut=`date -ud ''${vyyyy}-${vmm}-${vdd}' UTC '${vhh}':00:00' +%s` # convert time to universal time + + # Create necessary raw and proc directories + if [[ ! -d "$mrms_raw/${vyyyymmdd}" ]]; then + mkdir -p $mrms_raw/${vyyyymmdd} + fi + + # Check if file exists on disk; if not, pull it. + mrms_file="$mrms_proc/${vyyyymmdd}/MergedReflectivityQComposite_00.00_${vyyyy}${vmm}${vdd}-${vhh}0000.grib2" + echo "MRMS FILE:${mrms_file}" + + if [[ ! -f "${mrms_file}" ]]; then + cd $mrms_raw/${vyyyymmdd} + + # Name of MRMS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. + if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20200303 ]]; then + CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" >& /dev/null &` + Status=$? + if [[ ${Status} == 0 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" + else + CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" >& /dev/null &` + Status=$? + if [[ ${Status} == 0 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" + else + echo "ERROR: MRMR data not available for ${vyyyy}${vmm}${vdd}!" + exit + fi + fi + fi + + if [[ ${vyyyymmdd} -ge 20200303 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_prod_ldmdata_obs.tar" + fi + + echo "TAR FILE:${TarFile}" + + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"MergedReflectivityQComposite_00.00_${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print $7}'\`" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "MergedReflectivityQComposite_00.00_${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print $7}'` + Status=$? + + if [[ ${Status} != 0 ]]; then + echo "WARNING: Bad return status (${Status}) for date \"${CurDate}\". Did you forget to run \"module load hpss\"?" + echo "WARNING: ${TarCommand}" + else + if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then + mkdir -p $mrms_proc/${vyyyymmdd} + fi + + hour=0 + while [[ ${hour} -le 23 ]]; do + echo "hour=${hour}" + python ${SCRIPTSDIR}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} + hour=$((${hour} + 1)) # hourly increment + done + fi + + else + # Check if file exists on disk; if not, pull it. + mrms_file="$mrms_proc/${vyyyymmdd}/MergedReflectivityQComposite_00.00_${vyyyy}${vmm}${vdd}-${vhh}0000.grib2" + + if [[ ! -f "${mrms_file}" ]]; then + cd $mrms_raw/${vyyyymmdd} + + python ${SCRIPTSDIR}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${vhh} ${mrms_proc} ${mrms_raw} + fi + fi + + # Increment + current_fcst=$((${current_fcst} + 1)) # hourly increment + fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds + cur_ut=`expr ${start_valid_ut} + ${fcst_sec}` + +done diff --git a/scripts/exregional_get_ndas_files.sh b/scripts/exregional_get_ndas_files.sh new file mode 100755 index 0000000000..f79acf77f2 --- /dev/null +++ b/scripts/exregional_get_ndas_files.sh @@ -0,0 +1,145 @@ +#!/bin/sh + +# This script reorganizes the NDAS data into a more intuitive structure: +# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. + +# Top-level NDAS directory +ndas_dir=${OBS_DIR}/.. +if [[ ! -d "$ndas_dir" ]]; then + mkdir -p $ndas_dir +fi + +# NDAS data from HPSS +ndas_raw=$ndas_dir/raw +if [[ ! -d "$ndas_raw" ]]; then + mkdir -p $ndas_raw +fi + +# Reorganized NDAS location +ndas_proc=$ndas_dir/proc +if [[ ! -d "$ndas_proc" ]]; then + mkdir -p $ndas_proc +fi + +# Initialization +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh + +init=${CDATE}${hh} + +# Forecast length +fhr_last=`echo ${FHR} | awk '{ print $NF }'` + +fcst_length=${fhr_last} + +current_fcst=00 +while [[ ${current_fcst} -le ${fcst_length} ]]; do + fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds + yyyy=`echo ${init} | cut -c1-4` # year (YYYY) of initialization time + mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time + dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time + hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time + init_ut=`date -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time + vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time + vdate=`date -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time + vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time + vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time + vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) + +echo "yyyy mm dd hh= $yyyy $mm $dd $hh" +echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" + + vdate_ut_m1h=`expr ${vdate_ut} - 3600` # calculate current forecast time in universal time + vdate_m1h=`date -ud '1970-01-01 UTC '${vdate_ut_m1h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd_m1h=`echo ${vdate_m1h} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy_m1h=`echo ${vdate_m1h} | cut -c1-4` # year (YYYY) of valid time + vmm_m1h=`echo ${vdate_m1h} | cut -c5-6` # month (MM) of valid time + vdd_m1h=`echo ${vdate_m1h} | cut -c7-8` # day (DD) of valid time + vhh_m1h=`echo ${vdate_m1h} | cut -c9-10` # forecast hour (HH) + + vdate_ut_m2h=`expr ${vdate_ut} - 7200` # calculate current forecast time in universal time + vdate_m2h=`date -ud '1970-01-01 UTC '${vdate_ut_m2h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd_m2h=`echo ${vdate_m2h} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy_m2h=`echo ${vdate_m2h} | cut -c1-4` # year (YYYY) of valid time + vmm_m2h=`echo ${vdate_m2h} | cut -c5-6` # month (MM) of valid time + vdd_m2h=`echo ${vdate_m2h} | cut -c7-8` # day (DD) of valid time + vhh_m2h=`echo ${vdate_m2h} | cut -c9-10` # forecast hour (HH) + + vdate_ut_m3h=`expr ${vdate_ut} - 10800` # calculate current forecast time in universal time + vdate_m3h=`date -ud '1970-01-01 UTC '${vdate_ut_m3h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd_m3h=`echo ${vdate_m3h} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy_m3h=`echo ${vdate_m3h} | cut -c1-4` # year (YYYY) of valid time + vmm_m3h=`echo ${vdate_m3h} | cut -c5-6` # month (MM) of valid time + vdd_m3h=`echo ${vdate_m3h} | cut -c7-8` # day (DD) of valid time + vhh_m3h=`echo ${vdate_m3h} | cut -c9-10` # forecast hour (HH) + + vdate_ut_m4h=`expr ${vdate_ut} - 14400` # calculate current forecast time in universal time + vdate_m4h=`date -ud '1970-01-01 UTC '${vdate_ut_m4h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd_m4h=`echo ${vdate_m4h} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy_m4h=`echo ${vdate_m4h} | cut -c1-4` # year (YYYY) of valid time + vmm_m4h=`echo ${vdate_m4h} | cut -c5-6` # month (MM) of valid time + vdd_m4h=`echo ${vdate_m4h} | cut -c7-8` # day (DD) of valid time + vhh_m4h=`echo ${vdate_m4h} | cut -c9-10` # forecast hour (HH) + + vdate_ut_m5h=`expr ${vdate_ut} - 18000` # calculate current forecast time in universal time + vdate_m5h=`date -ud '1970-01-01 UTC '${vdate_ut_m5h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vyyyymmdd_m5h=`echo ${vdate_m5h} | cut -c1-8` # forecast time (YYYYMMDD) + vyyyy_m5h=`echo ${vdate_m5h} | cut -c1-4` # year (YYYY) of valid time + vmm_m5h=`echo ${vdate_m5h} | cut -c5-6` # month (MM) of valid time + vdd_m5h=`echo ${vdate_m5h} | cut -c7-8` # day (DD) of valid time + vhh_m5h=`echo ${vdate_m5h} | cut -c9-10` # forecast hour (HH) + + vhh_noZero=$(expr ${vhh} + 0) + +echo "vyyyymmdd_m1h vhh_m1h=$vyyyymmdd_m1h $vhh_m1h" +echo "vhh_noZero=$vhh_noZero" + + # Check if file exists on disk + ndas_file="$ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh}" + echo "NDAS PB FILE:${ndas_file}" + + if [[ ! -f "${ndas_file}" ]]; then + if [[ ! -d "$ndas_raw/${vyyyymmdd}${vhh}" ]]; then + mkdir -p $ndas_raw/${vyyyymmdd}${vhh} + fi + cd $ndas_raw/${vyyyymmdd}${vhh} + + # Name of NDAS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until July 2020. + if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -le 20190820 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + elif [[ ${vyyyymmdd} -ge 20190821 && ${vyyyymmdd} -le 20200226 ]]; then + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + else + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" + TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print $7}'\`" + echo "CALLING: ${TarCommand}" + htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print $7}'` + fi + + if [[ ! -d "$ndas_proc" ]]; then + mkdir -p $ndas_proc + fi + + if [[ ${vhh_noZero} -eq 0 || ${vhh} -eq 6 || ${vhh} -eq 12 || ${vhh} -eq 18 ]]; then + #echo "$ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm00.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh}" + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm00.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh} + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm01.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m1h}${vhh_m1h} + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm02.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m2h}${vhh_m2h} + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm03.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m3h}${vhh_m3h} + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm04.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m4h}${vhh_m4h} + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm05.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m5h}${vhh_m5h} + fi + fi + current_fcst=$((${current_fcst} + 6)) + echo "new fcst=${current_fcst}" + +done diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh new file mode 100755 index 0000000000..ac0c525d5a --- /dev/null +++ b/scripts/exregional_run_gridstatvx.sh @@ -0,0 +1,177 @@ +#!/bin/sh -l +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for grid-stat on +the UPP output files by initialization time for all forecast hours. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" "postprd_dir" "vx_dir" "gridstat_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args + +#----------------------------------------------------------------------- +# +# Remove any files from previous runs and stage necessary files in gridstat_dir. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting grid-stat verification" + +cd ${gridstat_dir} + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." + exit +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML +# +#----------------------------------------------------------------------- +# +export SCRIPTSDIR +export EXPTDIR +export MET_INSTALL_DIR +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export OBS_DIR +export VAR +export MODEL +export NET + +# +#----------------------------------------------------------------------- +# +# Run METplus +# +#----------------------------------------------------------------------- +# +if [ ${VAR} == "APCP" ]; then + export acc="${ACCUM}h" # for stats output prefix in GridStatConfig + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/${VAR}_${acc}.conf +elif [ ${VAR} == "REFC" ]; then + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/${VAR}.conf +else + echo "No variable defined" +fi + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus grid-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh new file mode 100755 index 0000000000..f06bb3c239 --- /dev/null +++ b/scripts/exregional_run_pointstatvx.sh @@ -0,0 +1,161 @@ +#!/bin/sh -l +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for point-stat on +the UPP output files by initialization time for all forecast hours. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" "postprd_dir" "vx_dir" "pointstat_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args +#----------------------------------------------------------------------- +# +# Remove any files from previous runs and stage necessary files in pointstat_dir. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting point-stat verification" + +cd ${pointstat_dir} + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML and run METplus +# +#----------------------------------------------------------------------- +# +export EXPTDIR +export MET_INSTALL_DIR +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export OBS_DIR +export MODEL +export NET + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_conus_sfc.conf + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_upper_air.conf + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus point-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/mrms_pull_topofhour.py b/scripts/mrms_pull_topofhour.py new file mode 100644 index 0000000000..072441643c --- /dev/null +++ b/scripts/mrms_pull_topofhour.py @@ -0,0 +1,103 @@ +import sys, os, shutil, subprocess +import datetime +import re, csv, glob +import bisect +import numpy as np + + +# Copy and unzip MRMS files that are closest to top of hour +# Done every hour on a 20-minute lag + +# Include option to define valid time on command line +# Used to backfill verification +#try: +valid_time = str(sys.argv[1]) + +YYYY = int(valid_time[0:4]) +MM = int(valid_time[4:6]) +DD = int(valid_time[6:8]) +HH = int(valid_time[8:19]) + +valid = datetime.datetime(YYYY,MM,DD,HH,0,0) + +#except IndexError: +# valid_time = None + + +# Default to current hour if not defined on command line +#if valid_time is None: +# now = datetime.datetime.utcnow() +# YYYY = int(now.strftime('%Y')) +# MM = int(now.strftime('%m')) +# DD = int(now.strftime('%d')) +# HH = int(now.strftime('%H')) + +# valid = datetime.datetime(YYYY,MM,DD,HH,0,0) +# valid_time = valid.strftime('%Y%m%d%H') + + +print('Pulling '+valid_time+' MRMS data') + + + + +# Set up working directory +DATA_HEAD = str(sys.argv[2]) +MRMS_PROD_DIR = str(sys.argv[3]) + +VALID_DIR = os.path.join(DATA_HEAD,valid.strftime('%Y%m%d')) +if not os.path.exists(VALID_DIR): + os.makedirs(VALID_DIR) +os.chdir(DATA_HEAD) + + + +# Copy and unzip the following MRMS products +#MRMS_PRODUCTS = ['MergedReflectivityQCComposite','SeamlessHSR','EchoTop'] +MRMS_PRODUCTS = ['MergedReflectivityQComposite'] + +for MRMS_PRODUCT in MRMS_PRODUCTS: + + if MRMS_PRODUCT == 'MergedReflectivityQComposite': + level = '_00.00_' + #elif MRMS_PRODUCT == 'SeamlessHSR': + #level = '_00.00_' + #elif MRMS_PRODUCT == 'EchoTop': + #level = '_18_00.50_' + + # Sort list of files for each MRMS product + print(valid.strftime('%Y%m%d')) + if valid.strftime('%Y%m%d') < '20200304': + search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' + elif valid.strftime('%Y%m%d') >= '20200304': + search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' + file_list = [f for f in glob.glob(search_path)] + time_list = [file_list[x][-24:-9] for x in range(len(file_list))] + int_list = [int(time_list[x][0:8]+time_list[x][9:15]) for x in range(len(time_list))] + int_list.sort() + datetime_list = [datetime.datetime.strptime(str(x),"%Y%m%d%H%M%S") for x in int_list] + + # Find the MRMS file closest to the valid time + i = bisect.bisect_left(datetime_list,valid) + closest_timestamp = min(datetime_list[max(0, i-1): i+2], key=lambda date: abs(valid - date)) + + # Check to make sure closest file is within +/- 15 mins of top of the hour + # Copy and rename the file for future ease + difference = abs(closest_timestamp - valid) + if difference.total_seconds() <= 900: + filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz' + filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz' + + if valid.strftime('%Y%m%d') < '20200304': + print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + + os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + os.system('gunzip '+VALID_DIR+'/'+filename2) + elif valid.strftime('%Y%m%d') >= '20200304': + print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + + os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + os.system('gunzip '+VALID_DIR+'/'+filename2) + else: + print('No '+MRMS_PRODUCT+' file found within 15 minutes of '+valid.strftime('%HZ %m/%d/%Y')+'. Skipping this time.') + diff --git a/ush/config.community.sh b/ush/config.community.sh index 34560bee09..a1000ed905 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -26,6 +26,22 @@ FV3GFS_FILE_FMT_LBCS="grib2" WTIME_RUN_FCST="01:00:00" +MODEL="FV3_GFS_v15p2_CONUS_25km" +METPLUS_PATH="path/to/METPlus" +MET_INSTALL_DIR="path/to/MET" +CCPA_OBS_DIR="/path/to/processed/CCPA/data" +MRMS_OBS_DIR="/path/to/processed/MRMS/data" +NDAS_OBS_DIR="/path/to/processed/NDAS/data" + +RUN_TASK_MAKE_GRID="TRUE" +RUN_TASK_MAKE_OROG="TRUE" +RUN_TASK_MAKE_SFC_CLIMO="TRUE" +RUN_TASK_GET_OBS_CCPA="FALSE" +RUN_TASK_GET_OBS_MRMS="FALSE" +RUN_TASK_GET_OBS_NDAS="FALSE" +RUN_TASK_VX_GRIDSTAT="FALSE" +RUN_TASK_VX_POINTSTAT="FALSE" + # # Uncomment the following line in order to use user-staged external model # files with locations and names as specified by EXTRN_MDL_SOURCE_BASEDIR_ICS/ @@ -38,6 +54,6 @@ WTIME_RUN_FCST="01:00:00" # EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="/path/to/model_data/FV3GFS" EXTRN_MDL_FILES_LBCS=( "gfs.pgrb2.0p25.f006" "gfs.pgrb2.0p25.f012" "gfs.pgrb2.0p25.f018" "gfs.pgrb2.0p25.f024" \ "gfs.pgrb2.0p25.f030" "gfs.pgrb2.0p25.f036" "gfs.pgrb2.0p25.f042" "gfs.pgrb2.0p25.f048" ) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 44c1156233..f0336561d7 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -394,6 +394,104 @@ SUB_HOURLY_POST="FALSE" DT_SUBHOURLY_POST_MNTS="00" #----------------------------------------------------------------------- # +# Set METplus parameters. Definitions: +# +# MODEL: +# String that specifies a descriptive name for the model being verified. +# +# MET_INSTALL_DIR: +# Location to top-level directory of MET installation. +# +# METPLUS_PATH: +# Location to top-level directory of METplus installation. +# +# CCPA_OBS_DIR: +# User-specified location of top-level directory where CCPA hourly +# precipitation files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via +# the get_obs_ccpa_tn task (activated in workflow by setting  +# RUN_TASK_GET_OBS_CCPA="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/ccpa/proc. METplus is configured to verify 01-, +# 03-, 06-, and 24-h accumulated precipitation using hourly CCPA files.  +# METplus configuration files require the use of predetermined directory +# structure and file names. Therefore, if the CCPA files are user +# provided, they need to follow the anticipated naming structure: +# {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, where YYYY is the 4-digit +# valid year, MM the 2-digit valid month, DD the 2-digit valid day of +# the month, and HH the 2-digit valid hour of the day. In addition, a +# caveat is noted for using hourly CCPA data. There is a problem with +# the valid time in the metadata for files valid from 19 - 00 UTC (or +# files  under the '00' directory). The script to pull the CCPA data +# from the NOAA HPSS has an example of how to account for this as well +# as organizing the data into a more intuitive format: +# regional_workflow/scripts/exregional_get_ccpa_files.sh. When a fix +# is provided, it will be accounted for in the +# exregional_get_ccpa_files.sh script. +# +# MRMS_OBS_DIR: +# User-specified location of top-level directory where MRMS composite +# reflectivity files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via the +# get_obs_mrms_tn task (activated in workflow by setting  +# RUN_TASK_GET_OBS_MRMS="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/mrms/proc. METplus configuration files require the +# use of predetermined directory structure and file names. Therefore, if +# the MRMS files are user provided, they need to follow the anticipated +# naming structure: +# {YYYYMMDD}/MergedReflectivityQComposite_00.00_{YYYYMMDD}-{HH}{mm}{SS}.grib2, +# where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD +# the 2-digit valid day of the month, HH the 2-digit valid hour of the +# day, mm the 2-digit valid minutes of the hour, and SS is the two-digit +# valid seconds of the hour. In addition, METplus is configured to look +# for a MRMS composite reflectivity file for the valid time of the +# forecast being verified; since MRMS composite reflectivity files do +# not always exactly match the valid time, a script, within the main +# script to retrieve MRMS data from the NOAA HPSS, is used to identify +# and rename the MRMS composite reflectivity file to match the valid +# time of the forecast. The script to pull the MRMS data from the NOAA +# HPSS has an example of the expected file naming structure: +# regional_workflow/scripts/exregional_get_mrms_files.sh. This script +# calls the script used to identify the MRMS file closest to the valid +# time: regional_workflow/scripts/mrms_pull_topofhour.py. +# +# NDAS_OBS_DIR: +# User-specified location of top-level directory where NDAS prepbufr +# files used by METplus are located. This parameter needs to be set for +# both user-provided observations and for observations that are +# retrieved from the NOAA HPSS (if the user has access) via the +# get_obs_ndas_tn task (activated in workflow by setting  +# RUN_TASK_GET_OBS_NDAS="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/ndas/proc. METplus is configured to verify +# near-surface variables hourly and upper-air variables at times valid +# at 00 and 12 UTC with NDAS prepbufr files. METplus configuration files +# require the use of predetermined file names. Therefore, if the NDAS +# files are user provided, they need to follow the anticipated naming +# structure: prepbufr.ndas.{YYYYMMDDHH}, where YYYY is the 4-digit valid +# year, MM the 2-digit valid month, DD the 2-digit valid day of the +# month, and HH the 2-digit valid hour of the day. The script to pull +# the NDAS data from the NOAA HPSS has an example of how to rename the +# NDAS data into a more intuitive format with the valid time listed in +# the file name: regional_workflow/scripts/exregional_get_ndas_files.sh +# +#----------------------------------------------------------------------- +# +MODEL="" +MET_INSTALL_DIR="/path/to/MET" +METPLUS_PATH="/path/to/METPlus" +CCPA_OBS_DIR="/path/to/observation-directory/ccpa/proc" +MRMS_OBS_DIR="/path/to/observation-directory/mrms/proc" +NDAS_OBS_DIR="/path/to/observation-directory/ndas/proc" +# +#----------------------------------------------------------------------- +# # Set initial and lateral boundary condition generation parameters. # Definitions: # @@ -501,7 +599,6 @@ EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) # directory or the cycle directories under it. # #----------------------------------------------------------------------- -# CCPP_PHYS_SUITE="FV3_GFS_v15p2" # #----------------------------------------------------------------------- @@ -1009,6 +1106,14 @@ VERBOSE="TRUE" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the surface climatology generation task. # +# RUN_TASK_VX_GRIDSTAT: +# Flag that determines whether the grid-stat verification task is to be +# run. +# +# RUN_TASK_VX_POINTSTAT: +# Flag that determines whether the point-stat verification task is to be +# run. +# #----------------------------------------------------------------------- # RUN_TASK_MAKE_GRID="TRUE" @@ -1019,6 +1124,17 @@ OROG_DIR="/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO="TRUE" SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files" + +RUN_TASK_GET_OBS_CCPA="FALSE" + +RUN_TASK_GET_OBS_MRMS="FALSE" + +RUN_TASK_GET_OBS_NDAS="FALSE" + +RUN_TASK_VX_GRIDSTAT="FALSE" + +RUN_TASK_VX_POINTSTAT="FALSE" + # #----------------------------------------------------------------------- # @@ -1215,6 +1331,17 @@ MAKE_ICS_TN="make_ics" MAKE_LBCS_TN="make_lbcs" RUN_FCST_TN="run_fcst" RUN_POST_TN="run_post" +GET_OBS="get_obs" +GET_OBS_CCPA_TN="get_obs_ccpa" +GET_OBS_MRMS_TN="get_obs_mrms" +GET_OBS_NDAS_TN="get_obs_ndas" +VX_TN="run_vx" +VX_GRIDSTAT_TN="run_gridstatvx" +VX_GRIDSTAT_REFC_TN="run_gridstatvx_refc" +VX_GRIDSTAT_03h_TN="run_gridstatvx_03h" +VX_GRIDSTAT_06h_TN="run_gridstatvx_06h" +VX_GRIDSTAT_24h_TN="run_gridstatvx_24h" +VX_POINTSTAT_TN="run_pointstatvx" # # Number of nodes. # @@ -1227,6 +1354,11 @@ NNODES_MAKE_ICS="4" NNODES_MAKE_LBCS="4" NNODES_RUN_FCST="" # This is calculated in the workflow generation scripts, so no need to set here. NNODES_RUN_POST="2" +NNODES_GET_OBS_CCPA="1" +NNODES_GET_OBS_MRMS="1" +NNODES_GET_OBS_NDAS="1" +NNODES_VX_GRIDSTAT="1" +NNODES_VX_POINTSTAT="1" # # Number of MPI processes per node. # @@ -1239,6 +1371,11 @@ PPN_MAKE_ICS="12" PPN_MAKE_LBCS="12" PPN_RUN_FCST="24" # This may have to be changed depending on the number of threads used. PPN_RUN_POST="24" +PPN_GET_OBS_CCPA="1" +PPN_GET_OBS_MRMS="1" +PPN_GET_OBS_NDAS="1" +PPN_VX_GRIDSTAT="1" +PPN_VX_POINTSTAT="1" # # Walltimes. # @@ -1251,6 +1388,11 @@ WTIME_MAKE_ICS="00:30:00" WTIME_MAKE_LBCS="00:30:00" WTIME_RUN_FCST="04:30:00" WTIME_RUN_POST="00:15:00" +WTIME_GET_OBS_CCPA="00:45:00" +WTIME_GET_OBS_MRMS="00:45:00" +WTIME_GET_OBS_NDAS="02:00:00" +WTIME_VX_GRIDSTAT="02:00:00" +WTIME_VX_POINTSTAT="01:00:00" # # Maximum number of attempts. # @@ -1263,6 +1405,15 @@ MAXTRIES_MAKE_ICS="1" MAXTRIES_MAKE_LBCS="1" MAXTRIES_RUN_FCST="1" MAXTRIES_RUN_POST="1" +MAXTRIES_GET_OBS_CCPA="1" +MAXTRIES_GET_OBS_MRMS="1" +MAXTRIES_GET_OBS_NDAS="1" +MAXTRIES_VX_GRIDSTAT="1" +MAXTRIES_VX_GRIDSTAT_REFC="1" +MAXTRIES_VX_GRIDSTAT_03h="1" +MAXTRIES_VX_GRIDSTAT_06h="1" +MAXTRIES_VX_GRIDSTAT_24h="1" +MAXTRIES_VX_POINTSTAT="1" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 15a69392d2..84c9c1fa7f 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -184,6 +184,20 @@ settings="\ 'make_lbcs_tn': ${MAKE_LBCS_TN} 'run_fcst_tn': ${RUN_FCST_TN} 'run_post_tn': ${RUN_POST_TN} + 'get_obs_ccpa_tn': ${GET_OBS_CCPA_TN} + 'get_obs_ndas_tn': ${GET_OBS_NDAS_TN} + 'get_obs_mrms_tn': ${GET_OBS_MRMS_TN} + 'vx_tn': ${VX_TN} + 'vx_gridstat_tn': ${VX_GRIDSTAT_TN} + 'vx_gridstat_refc_tn': ${VX_GRIDSTAT_REFC_TN} + 'vx_gridstat_03h_tn': ${VX_GRIDSTAT_03h_TN} + 'vx_gridstat_06h_tn': ${VX_GRIDSTAT_06h_TN} + 'vx_gridstat_24h_tn': ${VX_GRIDSTAT_24h_TN} + 'vx_pointstat_tn': ${VX_POINTSTAT_TN} +# +# Entity used to load the module file for each GET_OBS_* task. +# + 'get_obs': ${GET_OBS} # # Number of nodes to use for each task. # @@ -196,6 +210,11 @@ settings="\ 'nnodes_make_lbcs': ${NNODES_MAKE_LBCS} 'nnodes_run_fcst': ${NNODES_RUN_FCST} 'nnodes_run_post': ${NNODES_RUN_POST} + 'nnodes_get_obs_ccpa': ${NNODES_GET_OBS_CCPA} + 'nnodes_get_obs_mrms': ${NNODES_GET_OBS_MRMS} + 'nnodes_get_obs_ndas': ${NNODES_GET_OBS_NDAS} + 'nnodes_vx_gridstat': ${NNODES_VX_GRIDSTAT} + 'nnodes_vx_pointstat': ${NNODES_VX_POINTSTAT} # # Number of cores used for a task # @@ -214,6 +233,12 @@ settings="\ 'ppn_make_lbcs': ${PPN_MAKE_LBCS} 'ppn_run_fcst': ${PPN_RUN_FCST} 'ppn_run_post': ${PPN_RUN_POST} + 'ppn_get_obs_ccpa': ${PPN_GET_OBS_CCPA} + 'ppn_get_obs_mrms': ${PPN_GET_OBS_MRMS} + 'ppn_get_obs_ndas': ${PPN_GET_OBS_NDAS} + 'ppn_vx_gridstat': ${PPN_VX_GRIDSTAT} + 'ppn_vx_pointstat': ${PPN_VX_POINTSTAT} + # # Maximum wallclock time for each task. # @@ -226,6 +251,11 @@ settings="\ 'wtime_make_lbcs': ${WTIME_MAKE_LBCS} 'wtime_run_fcst': ${WTIME_RUN_FCST} 'wtime_run_post': ${WTIME_RUN_POST} + 'wtime_get_obs_ccpa': ${WTIME_GET_OBS_CCPA} + 'wtime_get_obs_mrms': ${WTIME_GET_OBS_MRMS} + 'wtime_get_obs_ndas': ${WTIME_GET_OBS_NDAS} + 'wtime_vx_gridstat': ${WTIME_VX_GRIDSTAT} + 'wtime_vx_pointstat': ${WTIME_VX_POINTSTAT} # # Maximum number of tries for each task. # @@ -238,12 +268,26 @@ settings="\ 'maxtries_make_lbcs': ${MAXTRIES_MAKE_LBCS} 'maxtries_run_fcst': ${MAXTRIES_RUN_FCST} 'maxtries_run_post': ${MAXTRIES_RUN_POST} + 'maxtries_get_obs_ccpa': ${MAXTRIES_GET_OBS_CCPA} + 'maxtries_get_obs_mrms': ${MAXTRIES_GET_OBS_MRMS} + 'maxtries_get_obs_ndas': ${MAXTRIES_GET_OBS_NDAS} + 'maxtries_vx_gridstat': ${MAXTRIES_VX_GRIDSTAT} + 'maxtries_vx_gridstat_refc': ${MAXTRIES_VX_GRIDSTAT_REFC} + 'maxtries_vx_gridstat_03h': ${MAXTRIES_VX_GRIDSTAT_03h} + 'maxtries_vx_gridstat_06h': ${MAXTRIES_VX_GRIDSTAT_06h} + 'maxtries_vx_gridstat_24h': ${MAXTRIES_VX_GRIDSTAT_24h} + 'maxtries_vx_pointstat': ${MAXTRIES_VX_POINTSTAT} # # Flags that specify whether to run the preprocessing tasks. # 'run_task_make_grid': ${RUN_TASK_MAKE_GRID} 'run_task_make_orog': ${RUN_TASK_MAKE_OROG} 'run_task_make_sfc_climo': ${RUN_TASK_MAKE_SFC_CLIMO} + 'run_task_get_obs_ccpa': ${RUN_TASK_GET_OBS_CCPA} + 'run_task_get_obs_mrms': ${RUN_TASK_GET_OBS_MRMS} + 'run_task_get_obs_ndas': ${RUN_TASK_GET_OBS_NDAS} + 'run_task_vx_gridstat': ${RUN_TASK_VX_GRIDSTAT} + 'run_task_vx_pointstat': ${RUN_TASK_VX_POINTSTAT} # # Number of physical cores per node for the current machine. # @@ -253,6 +297,7 @@ settings="\ # 'jobsdir': $JOBSDIR 'logdir': $LOGDIR + 'scriptsdir': $SCRIPTSDIR 'cycle_basedir': ${CYCLE_BASEDIR} 'global_var_defns_fp': ${GLOBAL_VAR_DEFNS_FP} 'load_modules_run_task_fp': ${LOAD_MODULES_RUN_TASK_FP} @@ -274,6 +319,18 @@ settings="\ # 'fcst_len_hrs': ${FCST_LEN_HRS} # +# METPlus-specific information +# + 'model': ${MODEL} + 'met_install_dir': ${MET_INSTALL_DIR} + 'metplus_path': ${METPLUS_PATH} + 'vx_config_dir': ${VX_CONFIG_DIR} + 'metplus_conf': ${METPLUS_CONF} + 'met_config': ${MET_CONFIG} + 'ccpa_obs_dir': ${CCPA_OBS_DIR} + 'mrms_obs_dir': ${MRMS_OBS_DIR} + 'ndas_obs_dir': ${NDAS_OBS_DIR} +# # Ensemble-related parameters. # 'do_ensemble': ${DO_ENSEMBLE} diff --git a/ush/setup.sh b/ush/setup.sh index 92b57cc319..484c62ddc4 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -232,6 +232,46 @@ fi # #----------------------------------------------------------------------- # +# Make sure that RUN_TASK_VX_GRIDSTAT is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_RUN_TASK_VX_GRIDSTAT" +# +# Set RUN_TASK_VX_GRIDSTAT to either "TRUE" or "FALSE" so we don't have to +# consider other valid values later on. +# +RUN_TASK_VX_GRIDSTAT=${RUN_TASK_VX_GRIDSTAT^^} +if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_GRIDSTAT}" = "YES" ]; then + RUN_TASK_VX_GRIDSTAT="TRUE" +elif [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ] || \ + [ "${RUN_TASK_VX_GRIDSTAT}" = "NO" ]; then + RUN_TASK_VX_GRIDSTAT="FALSE" +fi +# +#----------------------------------------------------------------------- +# +# Make sure that RUN_TASK_VX_POINTSTAT is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_RUN_TASK_VX_POINTSTAT" +# +# Set RUN_TASK_VX_POINTSTAT to either "TRUE" or "FALSE" so we don't have to +# consider other valid values later on. +# +RUN_TASK_VX_POINTSTAT=${RUN_TASK_VX_POINTSTAT^^} +if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_POINTSTAT}" = "YES" ]; then + RUN_TASK_VX_POINTSTAT="TRUE" +elif [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ] || \ + [ "${RUN_TASK_VX_POINTSTAT}" = "NO" ]; then + RUN_TASK_VX_POINTSTAT="FALSE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that DO_SHUM is set to a valid value. # #----------------------------------------------------------------------- @@ -732,7 +772,11 @@ SRC_DIR="${SR_WX_APP_TOP_DIR}/src" PARMDIR="$HOMErrfs/parm" MODULES_DIR="$HOMErrfs/modulefiles" EXECDIR="${SR_WX_APP_TOP_DIR}/bin" +FIXrrfs="$HOMErrfs/fix" TEMPLATE_DIR="$USHDIR/templates" +VX_CONFIG_DIR="$TEMPLATE_DIR/parm" +METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" +MET_CONFIG="$TEMPLATE_DIR/parm/met" case $MACHINE in @@ -1661,6 +1705,47 @@ one above. Reset values are: print_info_msg "$msg" fi + + if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ]; then + + msg=" +When RUN_ENVIR is set to \"nco\", it is assumed that the verification +will not be run. + RUN_TASK_VX_GRIDSTAT = \"${RUN_TASK_VX_GRIDSTAT}\" +Resetting RUN_TASK_VX_GRIDSTAT to \"FALSE\" +Reset value is:" + + RUN_TASK_VX_GRIDSTAT="FALSE" + + msg="$msg"" + RUN_TASK_VX_GRIDSTAT = \"${RUN_TASK_VX_GRIDSTAT}\" +" + + print_info_msg "$msg" + + fi + + if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ]; then + + msg=" +When RUN_ENVIR is set to \"nco\", it is assumed that the verification +will not be run. + RUN_TASK_VX_POINTSTAT = \"${RUN_TASK_VX_POINTSTAT}\" +Resetting RUN_TASK_VX_POINTSTAT to \"FALSE\" +Reset value is:" + + RUN_TASK_VX_POINTSTAT="FALSE" + + msg="$msg"" + RUN_TASK_VX_POINTSTAT = \"${RUN_TASK_VX_POINTSTAT}\" +" + + print_info_msg "$msg" + + fi + # #----------------------------------------------------------------------- # @@ -2492,12 +2577,16 @@ SRC_DIR="$SRC_DIR" PARMDIR="$PARMDIR" MODULES_DIR="${MODULES_DIR}" EXECDIR="$EXECDIR" +FIXrrfs="$FIXrrfs" FIXam="$FIXam" FIXLAM="$FIXLAM" FIXgsm="$FIXgsm" COMROOT="$COMROOT" COMOUT_BASEDIR="${COMOUT_BASEDIR}" TEMPLATE_DIR="${TEMPLATE_DIR}" +VX_CONFIG_DIR="${VX_CONFIG_DIR}" +METPLUS_CONF="${METPLUS_CONF}" +MET_CONFIG="${MET_CONFIG}" UFS_WTHR_MDL_DIR="${UFS_WTHR_MDL_DIR}" UFS_UTILS_DIR="${UFS_UTILS_DIR}" SFC_CLIMO_INPUT_DIR="${SFC_CLIMO_INPUT_DIR}" diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index c1f2154814..09e52ca6c1 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -265,7 +265,6 @@ FV3_GFS_v15p2: fhcyc: 0.0 fhlwr: 3600.0 fhswr: 3600.0 - fhzero: 6.0 hybedmf: True iau_delthrs: !!python/none iaufhrs: !!python/none @@ -418,7 +417,6 @@ FV3_CPT_v0: fhcyc: 0.0 fhlwr: 3600.0 fhswr: 3600.0 - fhzero: 6.0 fprcp: 2 hybedmf: True iccn: False diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index b42380e726..e15f3bfef4 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,18 +22,33 @@ Parameters needed by the job scheduler. - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + @@ -52,10 +67,19 @@ Directories and files. --> + + + + + + + + + + + + + &RSRV_HPSS; + &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_CCPA" + {{ nnodes_get_obs_ccpa }}:ppn={{ ppn_get_obs_ccpa }} + {{ wtime_get_obs_ccpa }} + &NCORES_PER_NODE; + &GET_OBS_CCPA_TN; + &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + ACCUM01 + + + + + + +{% endif %} + +{% if run_task_get_obs_mrms %} + + + + + &RSRV_HPSS; + &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_MRMS" + {{ nnodes_get_obs_mrms }}:ppn={{ ppn_get_obs_mrms }} + {{ wtime_get_obs_mrms }} + &NCORES_PER_NODE; + &GET_OBS_MRMS_TN; + &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + SCRIPTSDIR&SCRIPTSDIR; + VARREFC + + + + + + +{% endif %} + +{% if run_task_get_obs_ndas %} + + + + + &RSRV_HPSS; + &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_NDAS" + {{ nnodes_get_obs_ndas }}:ppn={{ ppn_get_obs_ndas }} + {{ wtime_get_obs_ndas }} + &NCORES_PER_NODE; + &GET_OBS_NDAS_TN; + &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&NDAS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARREFC + + + + + + +{% endif %} + +{% if run_task_vx_gridstat %} + + + + + &RSRV_HPSS; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_TN; + &LOGDIR;/&VX_GRIDSTAT_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM01 + + + {% if run_task_get_obs_ccpa %} + + {% else %} + + {% endif %} + + + +{% endif %} + +{% if run_task_vx_gridstat %} + + + + &RSRV_HPSS; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_REFC_TN; + &LOGDIR;/&VX_GRIDSTAT_REFC_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARREFC + + + {% if run_task_get_obs_mrms %} + + {% else %} + + {% endif %} + + + +{% endif %} + +{% if run_task_vx_gridstat %} + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_03h_TN; + &LOGDIR;/&VX_GRIDSTAT_03h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM03 + + + + + + +{% endif %} + +{% if run_task_vx_gridstat %} + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_06h_TN; + &LOGDIR;/&VX_GRIDSTAT_06h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM06 + + + + + + +{% endif %} + +{% if run_task_vx_gridstat %} + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_24h_TN; + &LOGDIR;/&VX_GRIDSTAT_24h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM24 + + + + + + +{% endif %} + +{% if run_task_vx_pointstat %} + + + + &RSRV_HPSS; + + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_POINTSTAT" + {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} + {{ wtime_vx_pointstat }} + &NCORES_PER_NODE; + &VX_POINTSTAT_TN; + &LOGDIR;/&VX_POINTSTAT_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&NDAS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + + + {% if run_task_get_obs_ndas %} + + {% else %} + + {% endif %} + + + + +{% endif %} {% if do_ensemble %} {% endif %} - diff --git a/ush/templates/parm/met/GridStatConfig_APCP b/ush/templates/parm/met/GridStatConfig_APCP new file mode 100755 index 0000000000..55fbd3e97c --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_APCP @@ -0,0 +1,178 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = BUDGET; + width = 2; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = STAT; + ctc = STAT; + cts = STAT; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + nbrctc = NONE; + nbrcts = NONE; + nbrcnt = STAT; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFL b/ush/templates/parm/met/GridStatConfig_REFL new file mode 100644 index 0000000000..b3ebb13ecc --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_REFL @@ -0,0 +1,219 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = NEAREST; + width = 1; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = NONE; + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >0. ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fourier decomposition +// May be set separately in each "obs.field" entry +// +fourier = { + wave_1d_beg = []; + wave_1d_end = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Gradient statistics +// May be set separately in each "obs.field" entry +// +gradient = { + dx = [ 1 ]; + dy = [ 1 ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = NONE; + ctc = STAT; + cts = STAT; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + nbrctc = STAT; + nbrcts = STAT; + nbrcnt = STAT; + grad = NONE; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + fourier = FALSE; + gradient = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PB2NCConfig_conus_sfc b/ush/templates/parm/met/PB2NCConfig_conus_sfc new file mode 100644 index 0000000000..55ce1da731 --- /dev/null +++ b/ush/templates/parm/met/PB2NCConfig_conus_sfc @@ -0,0 +1,161 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// PB2NC configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// PrepBufr message type +// +message_type = ${PB2NC_MESSAGE_TYPE} ; + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + +// +// Mapping of input PrepBufr message types to output message types +// +message_type_map = []; + +// +// PrepBufr station ID +// +station_id = ${PB2NC_STATION_ID} ; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +mask = { + grid = "${PB2NC_GRID}"; + poly = "${PB2NC_POLY}"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +elevation_range = { + beg = -1000; + end = 100000; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation types +// +pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; + +in_report_type = []; + +instrument_type = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Vertical levels to retain +// +level_range = { + beg = 1; + end = 511; +} + +level_category = [0, 1, 4, 5, 6]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// BUFR variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty, process all available variables. +// +obs_bufr_var = ${OBS_BUFR_VAR_LIST}; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input BUFR variable names to output variables names. +// The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +// +obs_bufr_map = []; + +// +// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB +// abbreviations in the output. This default map is appended to obs_bufr_map. +// This should not typically be overridden. +// +obs_prefbufr_map = [ + { key = "HOVI"; val = "VIS"; }, + { key = "PMO"; val = "PRMSL"; }, + { key = "POB"; val = "PRES"; }, + { key = "QOB"; val = "SPFH"; }, + { key = "TOB"; val = "TMP"; }, + { key = "TDO"; val = "DPT"; }, + { key = "ZOB"; val = "HGT"; }, + { key = "UOB"; val = "UGRD"; }, + { key = "VOB"; val = "VGRD"; }, + { key = "D_DPT"; val = "DPT"; }, + { key = "D_WDIR"; val = "WDIR"; }, + { key = "D_WIND"; val = "WIND"; }, + { key = "D_RH"; val = "RH"; }, + { key = "D_MIXR"; val = "MIXR"; }, + { key = "D_PBL"; val = "PBL"; }, + { key = "D_CAPE"; val = "CAPE"; }, + { key = "TOCC"; val = "TCDC"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +quality_mark_thresh = 3; +event_stack_flag = TOP; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +time_summary = { + flag = ${TIME_SUMMARY_FLAG}; + raw_data = FALSE; + beg = ${TIME_SUMMARY_BEG}; + end = ${TIME_SUMMARY_END}; + step = 300; + width = 600; + grib_code = []; + obs_var = ${TIME_SUMMARY_VAR_NAMES}; + type = ${TIME_SUMMARY_TYPES}; + vld_freq = 0; + vld_thresh = 0.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PB2NCConfig_upper_air b/ush/templates/parm/met/PB2NCConfig_upper_air new file mode 100644 index 0000000000..cf29a52088 --- /dev/null +++ b/ush/templates/parm/met/PB2NCConfig_upper_air @@ -0,0 +1,157 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// PB2NC configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// PrepBufr message type +// +message_type = ${PB2NC_MESSAGE_TYPE} ; + +// +// Mapping of message type group name to comma-separated list of values +// Derive PRMSL only for SURFACE message types +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + +// +// Mapping of input PrepBufr message types to output message types +// +message_type_map = []; + +// +// PrepBufr station ID +// +station_id = ${PB2NC_STATION_ID} ; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation time window +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation retention regions +// +mask = { + grid = "${PB2NC_GRID}"; + poly = "${PB2NC_POLY}"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observing location elevation +// +elevation_range = { + beg = -1000; + end = 100000; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Observation types +// +pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; + +in_report_type = []; + +instrument_type = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Vertical levels to retain +// +level_range = { + beg = 1; + end = 511; +} + +level_category = [0, 1, 4, 5, 6]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// BUFR variable names to retain or derive. +// Use obs_bufr_map to rename variables in the output. +// If empty, process all available variables. +// +obs_bufr_var = ${OBS_BUFR_VAR_LIST}; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input BUFR variable names to output variables names. +// The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +// +obs_bufr_map = []; + +// +// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB +// abbreviations in the output. This default map is appended to obs_bufr_map. +// This should not typically be overridden. +// +obs_prefbufr_map = [ + { key = "POB"; val = "PRES"; }, + { key = "QOB"; val = "SPFH"; }, + { key = "TOB"; val = "TMP"; }, + { key = "TDO"; val = "DPT"; }, + { key = "ZOB"; val = "HGT"; }, + { key = "UOB"; val = "UGRD"; }, + { key = "VOB"; val = "VGRD"; }, + { key = "D_DPT"; val = "DPT"; }, + { key = "D_WDIR"; val = "WDIR"; }, + { key = "D_WIND"; val = "WIND"; }, + { key = "D_RH"; val = "RH"; }, + { key = "D_MIXR"; val = "MIXR"; }, + { key = "D_PRMSL"; val = "PRMSL"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +quality_mark_thresh = 3; +event_stack_flag = TOP; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Time periods for the summarization +// obs_var (string array) is added and works like grib_code (int array) +// when use_var_id is enabled and variable names are saved. +// +time_summary = { + flag = ${TIME_SUMMARY_FLAG}; + raw_data = FALSE; + beg = ${TIME_SUMMARY_BEG}; + end = ${TIME_SUMMARY_END}; + step = 300; + width = 600; + grib_code = []; + obs_var = ${TIME_SUMMARY_VAR_NAMES}; + type = ${TIME_SUMMARY_TYPES}; + vld_freq = 0; + vld_thresh = 0.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_conus_sfc b/ush/templates/parm/met/PointStatConfig_conus_sfc new file mode 100644 index 0000000000..ae1350cf45 --- /dev/null +++ b/ush/templates/parm/met/PointStatConfig_conus_sfc @@ -0,0 +1,259 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; +rank_corr_flag = FALSE; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; + }; + +obs = { + field = [ ${OBS_FIELD} ]; + }; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = ${POINT_STAT_MESSAGE_TYPE}; +sid_exc = []; +obs_quality = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = NEAREST; + match_month = TRUE; + match_day = TRUE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; +write_cdf_bins = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Land/Sea mask +// For LANDSF message types, only use forecast grid points where land = TRUE. +// For WATERSF message types, only use forecast grid points where land = FALSE. +// land_mask.flag may be set separately in each "obs.field" entry. +// +land_mask = { + flag = FALSE; + file_name = []; + field = { name = "LAND"; level = "L0"; } + regrid = { method = NEAREST; width = 1; } + thresh = eq1; +} + +// +// Topography +// For SURFACE message types, only use observations where the topo - station +// elevation difference meets the use_obs_thresh threshold. +// For the observations kept, when interpolating forecast data to the +// observation location, only use forecast grid points where the topo - station +// difference meets the interp_fcst_thresh threshold. +// topo_mask.flag may be set separately in each "obs.field" entry. +// +topo_mask = { + flag = FALSE; + file_name = []; + field = { name = "TOPO"; level = "L0"; } + regrid = { method = BILIN; width = 2; } + use_obs_thresh = ge-100&&le100; + interp_fcst_thresh = ge-50&&le50; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.field" entry +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.field" entry +// +mask = { + grid = ${POINT_STAT_GRID}; +// poly = [ +// "${FIXverif_global}/vx_mask_files/grid2obs/CONUS.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/EAST.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/WEST.nc" +// ]; + poly = ${POINT_STAT_POLY}; + sid = ${POINT_STAT_STATION_ID}; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.field" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// May be set separately in each "obs.field" entry +// +interp = { + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = BILIN; + width = 2; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// HiRA verification method +// May be set separately in each "obs.field" entry +// +hira = { + flag = FALSE; + width = [ 2, 3, 4, 5 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.field" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = STAT; + sal1l2 = NONE; + vl1l2 = STAT; + val1l2 = NONE; + vcnt = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_upper_air b/ush/templates/parm/met/PointStatConfig_upper_air new file mode 100644 index 0000000000..0dff52c03c --- /dev/null +++ b/ush/templates/parm/met/PointStatConfig_upper_air @@ -0,0 +1,262 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; +rank_corr_flag = FALSE; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; + }; + +obs = { + field = [ ${OBS_FIELD} ]; + }; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = ${POINT_STAT_MESSAGE_TYPE}; +sid_exc = []; +obs_quality = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = NEAREST; + match_month = TRUE; + match_day = TRUE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; +write_cdf_bins = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Land/Sea mask +// For LANDSF message types, only use forecast grid points where land = TRUE. +// For WATERSF message types, only use forecast grid points where land = FALSE. +// land_mask.flag may be set separately in each "obs.field" entry. +// +land_mask = { + flag = FALSE; + file_name = []; + field = { name = "LAND"; level = "L0"; } + regrid = { method = NEAREST; width = 1; } + thresh = eq1; +} + +// +// Topography +// For SURFACE message types, only use observations where the topo - station +// elevation difference meets the use_obs_thresh threshold. +// For the observations kept, when interpolating forecast data to the +// observation location, only use forecast grid points where the topo - station +// difference meets the interp_fcst_thresh threshold. +// topo_mask.flag may be set separately in each "obs.field" entry. +// +topo_mask = { + flag = FALSE; + file_name = []; + field = { name = "TOPO"; level = "L0"; } + regrid = { method = BILIN; width = 2; } + use_obs_thresh = ge-100&&le100; + interp_fcst_thresh = ge-50&&le50; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.field" entry +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.field" entry +// +mask = { + grid = ${POINT_STAT_GRID}; +// poly = [ +// "${FIXverif_global}/vx_mask_files/grid2obs/NA.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/SA.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/NH.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/SH.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/TRO.nc", +// "${FIXverif_global}/vx_mask_files/grid2obs/CONUS.nc" +// ]; + poly = ${POINT_STAT_POLY}; + sid = ${POINT_STAT_STATION_ID}; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.field" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// May be set separately in each "obs.field" entry +// +interp = { + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = BILIN; + width = 2; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// HiRA verification method +// May be set separately in each "obs.field" entry +// +hira = { + flag = FALSE; + width = [ 2, 3, 4, 5 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.field" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = STAT; + sal1l2 = NONE; + vl1l2 = STAT; + val1l2 = NONE; + vcnt = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V9.0.2"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/metplus/APCP_01h.conf b/ush/templates/parm/metplus/APCP_01h.conf new file mode 100644 index 0000000000..01b1edc58e --- /dev/null +++ b/ush/templates/parm/metplus/APCP_01h.conf @@ -0,0 +1,91 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# ANLYS +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = GridStat + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A01 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A01 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/APCP_03h.conf b/ush/templates/parm/metplus/APCP_03h.conf new file mode 100644 index 0000000000..21a7d059a1 --- /dev/null +++ b/ush/templates/parm/metplus/APCP_03h.conf @@ -0,0 +1,121 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# format of filenames +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# ANLYS +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = PcpCombine, GridStat + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A03 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A03 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA b/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA new file mode 100644 index 0000000000..3235ff6316 --- /dev/null +++ b/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA @@ -0,0 +1,108 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +OUTPUT_BASE = {ENV[EXPTDIR]} +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# format of filenames +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# ANLYS +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.03h.hrap.conus.gb2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = PcpCombine, GridStat + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = False + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr({ENV[ACCUM]},{ENV[LAST_HR]},{ENV[ACCUM]}) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A03 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A03 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + diff --git a/ush/templates/parm/metplus/APCP_06h.conf b/ush/templates/parm/metplus/APCP_06h.conf new file mode 100644 index 0000000000..20ec8ec31b --- /dev/null +++ b/ush/templates/parm/metplus/APCP_06h.conf @@ -0,0 +1,121 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# format of filenames +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# ANLYS +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = PcpCombine, GridStat + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A06 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A06 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA b/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA new file mode 100644 index 0000000000..d3d1a9f205 --- /dev/null +++ b/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA @@ -0,0 +1,108 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +OUTPUT_BASE = {ENV[EXPTDIR]} +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# format of filenames +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# ANLYS +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.06h.hrap.conus.gb2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = PcpCombine, GridStat + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = False + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr({ENV[ACCUM]},{ENV[LAST_HR]},{ENV[ACCUM]}) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A06 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A06 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + diff --git a/ush/templates/parm/metplus/APCP_24h.conf b/ush/templates/parm/metplus/APCP_24h.conf new file mode 100644 index 0000000000..046eca2b37 --- /dev/null +++ b/ush/templates/parm/metplus/APCP_24h.conf @@ -0,0 +1,121 @@ +# Grid to Grid Precipitation + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# format of filenames +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# ANLYS +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = PcpCombine, GridStat + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A24 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700,ge25.400 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A24 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf new file mode 100644 index 0000000000..86f903b507 --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -0,0 +1,172 @@ +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pb2nc + +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat + +[config] + +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig_conus_sfc + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_conus_sfc + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]} +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {MODEL}_{OBTYPE}_ADPSFC + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPSFC + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +BOTH_VAR1_NAME = TMP +BOTH_VAR1_LEVELS = Z2 + +BOTH_VAR2_NAME = DPT +BOTH_VAR2_LEVELS = Z2 + +BOTH_VAR3_NAME = UGRD +BOTH_VAR3_LEVELS = Z10 +BOTH_VAR3_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR4_NAME = VGRD +BOTH_VAR4_LEVELS = Z10 +BOTH_VAR4_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR5_NAME = PRMSL +BOTH_VAR5_LEVELS = Z0 + +BOTH_VAR6_NAME = TCDC +BOTH_VAR6_LEVELS = L0 +BOTH_VAR6_OPTIONS = GRIB_lvl_typ = 200; + +BOTH_VAR7_NAME = VIS +BOTH_VAR7_LEVELS = L0 +BOTH_VAR7_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 + +FCST_VAR8_NAME = HGT +FCST_VAR8_LEVELS = L0 +FCST_VAR8_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; +FCST_VAR8_THRESH = <152, <305, <914, <1520, <3040, >=914 +OBS_VAR8_NAME = CEILING +OBS_VAR8_LEVELS = L0 +OBS_VAR8_OPTIONS = GRIB_lvl_typ = 215; +OBS_VAR8_THRESH = <152, <305, <914, <1520, <3040, >=914 + +FCST_VAR9_NAME = CAPE +FCST_VAR9_LEVELS = L0 +FCST_VAR9_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 +OBS_VAR9_NAME = CAPE +OBS_VAR9_LEVELS = L0-100000 +OBS_VAR9_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 + +FCST_VAR10_NAME = HGT +FCST_VAR10_LEVELS = L0 +FCST_VAR10_OPTIONS = GRIB_lvl_typ = 220; +OBS_VAR10_NAME = PBL +OBS_VAR10_LEVELS = L0 +OBS_VAR10_OPTIONS = desc = "RI"; + +BOTH_VAR11_NAME = GUST +BOTH_VAR11_LEVELS = Z0 diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf new file mode 100644 index 0000000000..fe86b17a91 --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -0,0 +1,157 @@ +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pb2nc + +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat +[config] + +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) +#LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig_upper_air + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, D_RH, D_CAPE, D_PBL + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_upper_air + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]} +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {MODEL}_{OBTYPE}_ADPUPA + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPUPA + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +BOTH_VAR1_NAME = TMP +BOTH_VAR1_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 + +BOTH_VAR2_NAME = RH +BOTH_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300 + +BOTH_VAR3_NAME = UGRD +BOTH_VAR3_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 +BOTH_VAR3_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR4_NAME = VGRD +BOTH_VAR4_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 +BOTH_VAR4_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR5_NAME = HGT +BOTH_VAR5_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 + +FCST_VAR6_NAME = CAPE +FCST_VAR6_LEVELS = L0 +FCST_VAR6_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR6_THRESH = >500, >1000, >1500, >2000, >3000, >4000 +OBS_VAR6_NAME = CAPE +OBS_VAR6_LEVELS = L0-100000 +OBS_VAR6_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR6_THRESH = >500, >1000, >1500, >2000, >3000, >4000 + +FCST_VAR7_NAME = HPBL +FCST_VAR7_LEVELS = Z0 +OBS_VAR7_NAME = PBL +OBS_VAR7_LEVELS = L0 +OBS_VAR7_OPTIONS = desc = "TKE"; + +FCST_VAR8_NAME = HGT +FCST_VAR8_LEVELS = L0 +FCST_VAR8_OPTIONS = GRIB_lvl_typ = 220; +OBS_VAR8_NAME = PBL +OBS_VAR8_LEVELS = L0 +OBS_VAR8_OPTIONS = desc = "RI"; diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf new file mode 100644 index 0000000000..db4864043a --- /dev/null +++ b/ush/templates/parm/metplus/REFC.conf @@ -0,0 +1,94 @@ +# Composite Reflectivity Verification + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# ANLYS +#OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/MergedReflectivityQComposite_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQComposite_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Model to verify +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# List of applications to run +PROCESS_LIST = GridStat + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFL +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Format of INIT_BEG and INIT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# Forecast/Observation variable Information +FCST_VAR1_NAME = REFC +FCST_VAR1_LEVELS = L0 + +BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 + +OBS_VAR1_NAME = MergedReflectivityQComposite +OBS_VAR1_LEVELS = L0 +OBS_VAR1_OPTIONS = censor_thresh = eq-999; censor_val = -9999; + +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# Neighborhood shape and width +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 1, 3, 5, 7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/common.conf b/ush/templates/parm/metplus/common.conf new file mode 100644 index 0000000000..fcdeeb9c2d --- /dev/null +++ b/ush/templates/parm/metplus/common.conf @@ -0,0 +1,22 @@ +[dir] +# Commonly used base METplus variables +# Location of METplus code +METPLUS_BASE = {ENV[METPLUS_PATH]} + +# Met install location +MET_INSTALL_DIR = {ENV[MET_INSTALL_DIR]} +MET_BASE = {MET_INSTALL_DIR}/share/met + +# Location of METplus parm directory +METPLUS_PARM_BASE = {ENV[METPLUS_CONF]} + +# Location of configuration files used by MET applications +CONFIG_DIR = {ENV[MET_CONFIG]} + +# Other directories +TMP_DIR = /tmp + +[config] +LOG_LEVEL = DEBUG +LOG_MET_VERBOSITY = 2 +LOG_MET_OUTPUT_TO_METPLUS = yes diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index dce4752a8b..f9f4f7c6d7 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -46,6 +46,8 @@ valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_la valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_MAKE_OROG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_MAKE_SFC_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_RUN_TASK_VX_GRIDSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_RUN_TASK_VX_POINTSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From c6feec4babe040ec6fbac8a7f9d9251d9c310080 Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Tue, 23 Mar 2021 14:17:30 -0600 Subject: [PATCH 026/203] Updating develop branch with updates from the release branch (#457) * Updating develop branch with updates from the release branch mostly focused on python plotting but also including the license. * Changed named of post file read in to account for additional 00 (min placeholder) * Removed 00 to account for subhourly data as that will be handled separately --- LICENSE.md | 21 ++++++++++++ ush/Python/plot_allvars.py | 43 ++++++++++++++----------- ush/Python/plot_allvars_diff.py | 57 ++++++++++++++++++--------------- ush/Python/qsub_job.sh | 50 +++++++++++++++++++++++++++++ ush/Python/qsub_job_diff.sh | 54 +++++++++++++++++++++++++++++++ ush/Python/sq_job.sh | 11 ++++--- ush/Python/sq_job_diff.sh | 12 ++++--- 7 files changed, 193 insertions(+), 55 deletions(-) create mode 100644 LICENSE.md create mode 100755 ush/Python/qsub_job.sh create mode 100755 ush/Python/qsub_job_diff.sh diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000000..3ae879af33 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +## Overview + +The regional_workflow code incorporated in the Unified Forecast System (UFS) was jointly developed by the National Oceanic and Atmospheric Administration (NOAA), Cooperative Institute for Research in the Atmosphere (CIRA), Cooperative Institute for Research in Environmental Sciences (CIRES), University Corporation for Atmospheric Research (UCAR), and I. M. Systems Group (IMSG). + +As a work of the United States Government, this project is in the public domain within the United States. +NOAA (IMSG by assignment), CIRA, CIRES, and UCAR waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/). + +## CC0 1.0 Universal Summary + +This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). + +## No copyright + +The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission. + +## Other information +In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights. Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer. + +## Contributions to this project +All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. + diff --git a/ush/Python/plot_allvars.py b/ush/Python/plot_allvars.py index 626b941abb..3c12f58901 100755 --- a/ush/Python/plot_allvars.py +++ b/ush/Python/plot_allvars.py @@ -1,6 +1,6 @@ ################################################################################ #### Python Script Documentation Block -# +# # Script name: plot_allvars.py # Script description: Generates plots from FV3-LAM post processed grib2 output # over the CONUS @@ -9,7 +9,7 @@ # David Wright Org: University of Michigan # # Instructions: Make sure all the necessary modules can be imported. -# Five command line arguments are needed: +# Six command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour # 3. Ending forecast hour @@ -27,11 +27,11 @@ # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide # -# To create plots for forecast hours 20-24 from 5/7 00Z +# To create plots for forecast hours 20-24 from 5/7 00Z # cycle with hourly output: # python plot_allvars.py 2020050700 20 24 1 \ # /path/to/expt_dirs/experiment/name \ -# /path/to/base/cartopy/maps +# /path/to/base/cartopy/maps # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -228,7 +228,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): parser.add_argument("Path to experiment directory") parser.add_argument("Path to base directory of cartopy shapefiles") args = parser.parse_args() - + # Read date/time, forecast hour, and directory paths from command line ymdh = str(sys.argv[1]) ymd = ymdh[0:8] @@ -312,7 +312,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): print(Lon0) # Specify plotting domains -# User can add domains here, just need to specify lat/lon information below +# User can add domains here, just need to specify lat/lon information below # (if dom == 'conus' block) domains=['conus'] # Other option is 'regional' @@ -368,7 +368,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): qpf = data1.select(name='Total Precipitation',lengthOfTimeRange=fhr)[0].values * 0.0393701 # Composite reflectivity - refc = data1.select(name='Maximum/Composite radar reflectivity')[0].values + refc = data1.select(name='Maximum/Composite radar reflectivity')[0].values if (fhr > 0): # Max/Min Hourly 2-5 km Updraft Helicity @@ -391,8 +391,13 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): def main(): # Number of processes must coincide with the number of domains to plot - pool = multiprocessing.Pool(len(domains)) - pool.map(plot_all,domains) + #pool = multiprocessing.Pool(len(domains)) + #pool.map(plot_all,domains) + + # To avoid import multiprocessing recursively on MacOS etc. + # Anyway since we only have one domain for SRW application + for dom in domains: + plot_all(dom) def plot_all(dom): @@ -401,7 +406,7 @@ def plot_all(dom): # Map corners for each domain if dom == 'conus': llcrnrlon = -120.5 - llcrnrlat = 21.0 + llcrnrlat = 21.0 urcrnrlon = -64.5 urcrnrlat = 49.0 lat_0 = 35.4 @@ -421,7 +426,7 @@ def plot_all(dom): fig = plt.figure(figsize=(10,10)) ax1 = fig.add_axes([0.1,0.1,0.8,0.8]) - # Define where Cartopy Maps are located + # Define where Cartopy Maps are located cartopy.config['data_dir'] = CARTOPY_DIR back_res='50m' @@ -455,7 +460,7 @@ def plot_all(dom): linewidth=fline_wd,alpha=falpha) # All lat lons are earth relative, so setup the associated projection correct for that data - transform = ccrs.PlateCarree() + transform = ccrs.PlateCarree() # high-resolution background images if back_img=='on': @@ -484,7 +489,7 @@ def plot_all(dom): cm = plt.cm.Spectral_r norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - cs1_a = plt.pcolormesh(lon_shift,lat_shift,slp,transform=transform,cmap=cm,norm=norm) + cs1_a = plt.pcolormesh(lon_shift,lat_shift,slp,transform=transform,cmap=cm,norm=norm) cbar1 = plt.colorbar(cs1_a,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') cbar1.set_label(units,fontsize=8) cbar1.ax.tick_params(labelsize=8) @@ -583,7 +588,7 @@ def plot_all(dom): cbar1.ax.tick_params(labelsize=8) plt.barbs(lon_shift[::skip,::skip],lat_shift[::skip,::skip],uwind[::skip,::skip],vwind[::skip,::skip],length=barblength,linewidth=0.5,color='black',transform=transform) ax.text(.5,1.03,'FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - + compress_and_save(EXPT_DIR+'/'+ymdh+'/postprd/10mwind_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) @@ -603,13 +608,13 @@ def plot_all(dom): units = 'J/kg' clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] clevs2 = [-2000,-500,-250,-100,-25] - colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] + colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred','darkviolet'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) cs_1 = plt.pcolormesh(lon_shift,lat_shift,cape,transform=transform,cmap=cm,vmin=100,norm=norm) cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('darkviolet') + cs_1.cmap.set_over('black') cbar1 = plt.colorbar(cs_1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar1.set_label(units,fontsize=8) cbar1.ax.tick_params(labelsize=8) @@ -706,7 +711,7 @@ def plot_all(dom): units = 'in' clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] clevsdif = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] - colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) @@ -740,7 +745,7 @@ def plot_all(dom): colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - + cs_1 = plt.pcolormesh(lon_shift,lat_shift,refc,transform=transform,cmap=cm,vmin=5,norm=norm) cs_1.cmap.set_under('white',alpha=0.) cs_1.cmap.set_over('black') @@ -758,7 +763,7 @@ def plot_all(dom): ################################# # Plot Max/Min Hourly 2-5 km UH ################################# - if (fhr > 0): # Do not make max/min hourly 2-5 km UH plot for forecast hour 0 + if (fhr > 0): # Do not make max/min hourly 2-5 km UH plot for forecast hour 0 t1 = time.perf_counter() print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) diff --git a/ush/Python/plot_allvars_diff.py b/ush/Python/plot_allvars_diff.py index a44a147e9c..ae0d718c6b 100755 --- a/ush/Python/plot_allvars_diff.py +++ b/ush/Python/plot_allvars_diff.py @@ -1,8 +1,8 @@ ################################################################################ #### Python Script Documentation Block -# +# # Script name: plot_allvars_diff.py -# Script description: Generates difference plots from FV3-LAM post processed +# Script description: Generates difference plots from FV3-LAM post processed # grib2 output over the CONUS # # Authors: Ben Blake Org: NOAA/NWS/NCEP/EMC Date: 2020-08-24 @@ -30,11 +30,11 @@ # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide # -# To create plots for forecast hours 20-24 from 5/7 00Z +# To create plots for forecast hours 20-24 from 5/7 00Z # cycle with hourly output: -# python plot_allvars_diff.py 2020050700 20 24 1 \ +# python plot_allvars_diff.py 2020050700 20 24 1 \ # /path/to/expt_dir_1 /path/to/expt_dir_2 \ -# /path/to/base/cartopy/maps +# /path/to/base/cartopy/maps # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -232,7 +232,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): parser.add_argument("Path to experiment 2 directory") parser.add_argument("Path to base directory of cartopy shapefiles") args = parser.parse_args() - + # Read date/time, forecast hour, and directory paths from command line ymdh = str(sys.argv[1]) ymd = ymdh[0:8] @@ -323,7 +323,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): print(Lon0) # Specify plotting domains -# User can add domains here, just need to specify lat/lon information below +# User can add domains here, just need to specify lat/lon information below # (if dom == 'conus' block) domains=['conus'] # Other option is 'regional' @@ -413,8 +413,8 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): qpf_diff = qpf_2 - qpf_1 # Composite reflectivity - refc_1 = data1.select(name='Maximum/Composite radar reflectivity')[0].values - refc_2 = data2.select(name='Maximum/Composite radar reflectivity')[0].values + refc_1 = data1.select(name='Maximum/Composite radar reflectivity')[0].values + refc_2 = data2.select(name='Maximum/Composite radar reflectivity')[0].values if (fhr > 0): # Max/Min Hourly 2-5 km Updraft Helicity @@ -442,9 +442,14 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): def main(): - # Number of processes must coincide with the number of domains to plot - pool = multiprocessing.Pool(len(domains)) - pool.map(plot_all,domains) + # Number of processes must coincide with the number of domains to plot + #pool = multiprocessing.Pool(len(domains)) + #pool.map(plot_all,domains) + + # To avoid import multiprocessing recursively on MacOS/Windows etc. + # Anyway since we only have one domain for SRW application + for dom in domains: + plot_all(dom) def plot_all(dom): @@ -453,7 +458,7 @@ def plot_all(dom): # Map corners for each domain if dom == 'conus': llcrnrlon = -120.5 - llcrnrlat = 21.0 + llcrnrlat = 21.0 urcrnrlon = -64.5 urcrnrlat = 49.0 lat_0 = 35.4 @@ -473,7 +478,7 @@ def plot_all(dom): fig = plt.figure(figsize=(10,10)) gs = GridSpec(9,9,wspace=0.0,hspace=0.0) - # Define where Cartopy Maps are located + # Define where Cartopy Maps are located cartopy.config['data_dir'] = CARTOPY_DIR back_res='50m' @@ -511,7 +516,7 @@ def plot_all(dom): linewidth=fline_wd,alpha=falpha) # All lat lons are earth relative, so setup the associated projection correct for that data - transform = ccrs.PlateCarree() + transform = ccrs.PlateCarree() # high-resolution background images if back_img=='on': @@ -557,7 +562,7 @@ def plot_all(dom): norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) - cs1_a = ax1.pcolormesh(lon_shift,lat_shift,slp_1,transform=transform,cmap=cm,norm=norm) + cs1_a = ax1.pcolormesh(lon_shift,lat_shift,slp_1,transform=transform,cmap=cm,norm=norm) cbar1 = plt.colorbar(cs1_a,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') cbar1.set_label(units,fontsize=6) cbar1.ax.tick_params(labelsize=5) @@ -565,7 +570,7 @@ def plot_all(dom): plt.clabel(cs1_b,np.arange(940,1060,4),inline=1,fmt='%d',fontsize=6) ax1.text(.5,1.03,'FV3-LAM SLP ('+units+') \n initialized: '+itime+' valid: '+vtime + ' (f'+fhour+')',horizontalalignment='center',fontsize=8,transform=ax1.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,slp_2,transform=transform,cmap=cm,norm=norm) + cs2_a = ax2.pcolormesh(lon2_shift,lat2_shift,slp_2,transform=transform,cmap=cm,norm=norm) cbar2 = plt.colorbar(cs2_a,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,extend='both') cbar2.set_label(units,fontsize=6) cbar2.ax.tick_params(labelsize=5) @@ -738,7 +743,7 @@ def plot_all(dom): cbar3.set_label(units,fontsize=6) cbar3.ax.tick_params(labelsize=6) ax3.text(.5,1.03,'FV3-LAM-2 - FV3-LAM 10-m Winds ('+units+') \n initialized: '+itime+' valid: '+vtime+' (f'+fhour+')',horizontalalignment='center',fontsize=6,transform=ax3.transAxes,bbox=dict(facecolor='white',alpha=0.85,boxstyle='square,pad=0.2')) - + compress_and_save(EXPT_DIR_1+'/'+ymdh+'/postprd/10mwind_diff_'+dom+'_f'+fhour+'.png') t2 = time.perf_counter() t3 = round(t2-t1, 3) @@ -763,14 +768,14 @@ def plot_all(dom): clevs = [100,250,500,1000,1500,2000,2500,3000,3500,4000,4500,5000] clevs2 = [-2000,-500,-250,-100,-25] clevsdiff = [-2000,-1500,-1000,-500,-250,-100,0,100,250,500,1000,1500,2000] - colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred'] + colorlist = ['blue','dodgerblue','cyan','mediumspringgreen','#FAFAD2','#EEEE00','#EEC900','darkorange','crimson','darkred','darkviolet'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) cs_1 = ax1.pcolormesh(lon_shift,lat_shift,cape_1,transform=transform,cmap=cm,vmin=100,norm=norm) cs_1.cmap.set_under('white',alpha=0.) - cs_1.cmap.set_over('darkviolet') + cs_1.cmap.set_over('black') cbar1 = plt.colorbar(cs_1,ax=ax1,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar1.set_label(units,fontsize=6) cbar1.ax.tick_params(labelsize=4) @@ -779,7 +784,7 @@ def plot_all(dom): cs_2 = ax2.pcolormesh(lon2_shift,lat2_shift,cape_2,transform=transform,cmap=cm,vmin=100,norm=norm) cs_2.cmap.set_under('white',alpha=0.) - cs_2.cmap.set_over('darkviolet') + cs_2.cmap.set_over('black') cbar2 = plt.colorbar(cs_2,ax=ax2,orientation='horizontal',pad=0.05,shrink=0.6,ticks=clevs,extend='max') cbar2.set_label(units,fontsize=6) cbar2.ax.tick_params(labelsize=4) @@ -816,7 +821,7 @@ def plot_all(dom): units = 'x10${^5}$ s${^{-1}}$' skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') + print('skipping every '+str(skip)+' grid points to plot') barblength = 4 vortlevs = [16,20,24,28,32,36,40] @@ -878,7 +883,7 @@ def plot_all(dom): units = 'kts' skip = round(177.28*(dx/1000.)**-.97) - print('skipping every '+str(skip)+' grid points to plot') + print('skipping every '+str(skip)+' grid points to plot') barblength = 4 clevs = [50,60,70,80,90,100,110,120,130,140,150] @@ -938,7 +943,7 @@ def plot_all(dom): units = 'in' clevs = [0.01,0.1,0.25,0.5,0.75,1,1.25,1.5,1.75,2,2.5,3,4,5,7,10,15,20] clevsdiff = [-3,-2.5,-2,-1.5,-1,-0.5,0,0.5,1,1.5,2,2.5,3] - colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] + colorlist = ['chartreuse','limegreen','green','blue','dodgerblue','deepskyblue','cyan','mediumpurple','mediumorchid','darkmagenta','darkred','crimson','orangered','darkorange','goldenrod','gold','yellow'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) normdiff = matplotlib.colors.BoundaryNorm(clevsdiff, cmdiff.N) @@ -978,7 +983,7 @@ def plot_all(dom): ################################# # Plot Max/Min Hourly 2-5 km UH ################################# -# Do not make max/min hourly 2-5 km UH plot for forecast hour 0 +# Do not make max/min hourly 2-5 km UH plot for forecast hour 0 t1 = time.perf_counter() print(('Working on Max/Min Hourly 2-5 km UH for '+dom)) @@ -1050,7 +1055,7 @@ def plot_all(dom): colorlist = ['turquoise','dodgerblue','mediumblue','lime','limegreen','green','#EEEE00','#EEC900','darkorange','red','firebrick','darkred','fuchsia'] cm = matplotlib.colors.ListedColormap(colorlist) norm = matplotlib.colors.BoundaryNorm(clevs, cm.N) - + cs_1 = ax1.pcolormesh(lon_shift,lat_shift,refc_1,transform=transform,cmap=cm,vmin=5,norm=norm) cs_1.cmap.set_under('white',alpha=0.) cs_1.cmap.set_over('black') diff --git a/ush/Python/qsub_job.sh b/ush/Python/qsub_job.sh new file mode 100755 index 0000000000..25e8814189 --- /dev/null +++ b/ush/Python/qsub_job.sh @@ -0,0 +1,50 @@ +#!/bin/sh +#PBS -A an_account +#PBS -q regular +#PBS -l select=1:mpiprocs=24:ncpus=24 +#PBS -l walltime=00:20:00 +#PBS -N plot_allvars +#PBS -j oe -o plot_allvars.out + +# Prior to submitting the script the following environment variables +# must be set using export or setenv +# HOMErrfs=/path-to/ufs-srweather-app/regional_workflow +# EXPTDIR=/path-to/expt_dirs/your_experiment + +cd ${HOMErrfs}/ush/Python +set -x + +source /etc/profile.d/modules.sh + +############ +# Python environment for Cheyenne +############ +ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics + +############ +# Path to shape files +############ +#Cheyenne: +SHAPE_FILES=/glade/p/ral/jntp/UFS_SRW_app/tools/NaturalEarth + +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export FCST_START=3 +export FCST_END=${FCST_LEN_HRS} +export FCST_INC=3 + +# Usage statement: Make sure all the necessary modules can be imported. +# Five command line arguments are needed: +# 1. Cycle date/time in YYYYMMDDHH format +# 2. Starting forecast hour in HHH format +# 3. Ending forecast hour in HHH format +# 4. Forecast hour increment +# 5. EXPT_DIR: Experiment directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR/YYYYMMDDHH/postprd/ +# 6. CARTOPY_DIR: Base directory of cartopy shapefiles +# -File structure should be: +# CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp + +python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR} ${SHAPE_FILES} diff --git a/ush/Python/qsub_job_diff.sh b/ush/Python/qsub_job_diff.sh new file mode 100755 index 0000000000..e0a6383863 --- /dev/null +++ b/ush/Python/qsub_job_diff.sh @@ -0,0 +1,54 @@ +#!/bin/sh +#PBS -A an_account +#PBS -q regular +#PBS -l select=1:mpiprocs=24:ncpus=24 +#PBS -l walltime=00:30:00 +#PBS -N plot_allvars_diff +#PBS -j oe -o plot_allvars_diff.out + +# Prior to submitting the script the following environment variables +# must be set using export or setenv +# HOMErrfs=/path-to/ufs-srweather-app/regional_workflow +# EXPTDIR1=/path-to/expt_dirs/your_experiment1 +# EXPTDIR2=/path-to/expt_dirs/your_experiment2 + +cd ${HOMErrfs}/ush/Python +set -x + +source /etc/profile.d/modules.sh + +############ +# Python environment for Cheyenne +############ +ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics + +############ +# Path to shape files +############ +#Cheyenne: +SHAPE_FILES=/glade/p/ral/jntp/UFS_SRW_app/tools/NaturalEarth + +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export FCST_START=3 +export FCST_END=${FCST_LEN_HRS} +export FCST_INC=3 + +# Usage statement: Make sure all the necessary modules can be imported. +# Seven command line arguments are needed: +# 1. Cycle date/time in YYYYMMDDHH format +# 2. Starting forecast hour in HHH format +# 3. Ending forecast hour in HHH format +# 4. Forecast hour increment +# 5. EXPT_DIR_1: Experiment 1 directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR_1/YYYYMMDDHH/postprd/ +# 6. EXPT_DIR_2: Experiment 2 directory +# -Postprocessed data should be found in the directory: +# EXPT_DIR_2/YYYYMMDDHH/postprd/ +# 7. CARTOPY_DIR: Base directory of cartopy shapefiles +# -File structure should be: +# CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp + +python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR1} ${EXPTDIR2} ${SHAPE_FILES} diff --git a/ush/Python/sq_job.sh b/ush/Python/sq_job.sh index ec8317ebb9..1c594efd46 100755 --- a/ush/Python/sq_job.sh +++ b/ush/Python/sq_job.sh @@ -35,19 +35,20 @@ conda activate pygraf ############ # Python environment for Gaea ############ -#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -#module load miniconda3 -#conda activate pygraf +#module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +#module load miniconda3/4.8.3-regional-workflow ############ # Path to shape files ############ #Hera: -SHAPE_FILES=/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/tools/NaturalEarth +SHAPE_FILES=/scratch2/BMC/det/UFS_SRW_app/v1p0/fix_files/NaturalEarth #Jet: +#SHAPE_FILES=/lfs4/BMC/wrfruc/FV3-LAM/NaturalEarth #Orion: -#SHAPE_FILES=/home/chjeon/tools/NaturalEarth +#SHAPE_FILES=/work/noaa/gsd-fv3-dev/UFS_SRW_App/v1p0/fix_files/NaturalEarth #Gaea: +#SHAPE_FILES=/lustre/f2/pdata/esrl/gsd/ufs/NaturalEarth export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" source ${GLOBAL_VAR_DEFNS_FP} diff --git a/ush/Python/sq_job_diff.sh b/ush/Python/sq_job_diff.sh index 9c49a7a1a7..84143ffd86 100755 --- a/ush/Python/sq_job_diff.sh +++ b/ush/Python/sq_job_diff.sh @@ -22,6 +22,7 @@ module load hpss ############ # Python environment for Jet and Hera +############ module use -a /contrib/miniconda3/modulefiles module load miniconda3 conda activate pygraf @@ -36,19 +37,20 @@ conda activate pygraf ############ # Python environment for Gaea ############ -#module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -#module load miniconda3 -#conda activate pygraf +#module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +#module load miniconda3/4.8.3-regional-workflow ############ # Path to shape files ############ #Hera: -SHAPE_FILES=/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/tools/NaturalEarth +SHAPE_FILES=/scratch2/BMC/det/UFS_SRW_app/v1p0/fix_files/NaturalEarth #Jet: +#SHAPE_FILES=/lfs4/BMC/wrfruc/FV3-LAM/NaturalEarth #Orion: -#SHAPE_FILES=/home/chjeon/tools/NaturalEarth +#SHAPE_FILES=/work/noaa/gsd-fv3-dev/UFS_SRW_App/v1p0/fix_files/NaturalEarth #Gaea: +#SHAPE_FILES=/lustre/f2/pdata/esrl/gsd/ufs/NaturalEarth export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" source ${GLOBAL_VAR_DEFNS_FP} From 36c75edf1bff0811905a9a1eedff26427a714253 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Fri, 26 Mar 2021 11:28:43 -0600 Subject: [PATCH 027/203] Rename GSD_RAP13km grid and adjust its write-component grid (#458) ## DESCRIPTION OF CHANGES: * Rename GSD_RAP13km grid to RRFS_NA_13km. * In ush/set_predef_grid_params.sh: * Change parameters of the (newly renamed) RRFS_NA_13km grid to make its write-component grid be completely within the native grid. * For the (newly renamed) RRFS_NA_13km grid, automate calculation of WRTCMP_dlon and WRTCMP_dlat from ESGgrid_DELX and ESGgrid_DELY, respectively. * NCL scripts: * Make modifications to make scripts more general and have the find_wrtcmp_grid_params.ncl script work for a write-component grid of type "rotated_latlon". * Remove unused code in NCL scripts. * Fix up comments. ## TESTS CONDUCTED: Ran the (renamed) WE2E test grid_RRFS_NA_13km on Hera. Was successful. ## CONTRIBUTORS (optional): This grid was also tested by @jaymes-kenyon on Jet. --- ...RAP13km.sh => config.grid_RRFS_NA_13km.sh} | 2 +- tests/baselines_list.txt | 2 +- ...calc_wrtcmp_grid_params_lambert_cnfrml.ncl | 4 +- ...calc_wrtcmp_grid_params_rotated_latlon.ncl | 144 +++++++++++-- ush/NCL/find_wrtcmp_grid_params.ncl | 34 +++- ush/NCL/plot_grid.ncl | 42 +++- ush/NCL/read_FV3LAM_grid_wrtcmp.ncl | 189 ++++++++++++++---- ush/set_gridparams_ESGgrid.sh | 3 +- ush/set_predef_grid_params.sh | 46 ++++- ush/valid_param_vals.sh | 2 +- 10 files changed, 385 insertions(+), 83 deletions(-) rename tests/baseline_configs/{config.grid_GSD_RAP13km.sh => config.grid_RRFS_NA_13km.sh} (90%) diff --git a/tests/baseline_configs/config.grid_GSD_RAP13km.sh b/tests/baseline_configs/config.grid_RRFS_NA_13km.sh similarity index 90% rename from tests/baseline_configs/config.grid_GSD_RAP13km.sh rename to tests/baseline_configs/config.grid_RRFS_NA_13km.sh index a3ce2ed2da..0c0ecbd052 100644 --- a/tests/baseline_configs/config.grid_GSD_RAP13km.sh +++ b/tests/baseline_configs/config.grid_RRFS_NA_13km.sh @@ -1,7 +1,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="GSD_RAP13km" +PREDEF_GRID_NAME="RRFS_NA_13km" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_GSD_v0" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 5405fce017..fcd7371b31 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -19,7 +19,6 @@ get_from_HPSS_ics_RAP_lbcs_RAP grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS grid_GSD_HRRR_AK_50km -grid_GSD_RAP13km grid_RRFS_AK_13km_FV3GFS_FV3GFS grid_RRFS_AK_13km_RAP_RAP grid_RRFS_AK_3km_FV3GFS_FV3GFS @@ -49,6 +48,7 @@ grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +grid_RRFS_NA_13km grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16beta grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR diff --git a/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl b/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl index c4c589b0df..07b45c261b 100644 --- a/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl +++ b/ush/NCL/calc_wrtcmp_grid_params_lambert_cnfrml.ncl @@ -244,7 +244,7 @@ begin param_value := char_sq + "lambert_conformal" + char_sq param_value_str := tostring(param_value) - regex_search = "^(\s*" + param_name + ":\s+)('\{\{ " + param_name + " \}\}')(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)('\{\{\s*" + param_name + "\s*\}\}')(\s*)(.*)" regex_print = "\1" + param_value_str sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ @@ -326,7 +326,7 @@ begin ; ; Set the regular expression to search for. ; - regex_search = "^(\s*" + param_name + ":\s+)(\{\{ " + param_name + " \}\})(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)(\{\{\s*" + param_name + "\s*\}\})(\s*)(.*)" ; ; Get the parameter name and value without the trailing comment (if any). ; diff --git a/ush/NCL/calc_wrtcmp_grid_params_rotated_latlon.ncl b/ush/NCL/calc_wrtcmp_grid_params_rotated_latlon.ncl index 250442161b..b74925d124 100644 --- a/ush/NCL/calc_wrtcmp_grid_params_rotated_latlon.ncl +++ b/ush/NCL/calc_wrtcmp_grid_params_rotated_latlon.ncl @@ -10,6 +10,7 @@ loadscript(lib_location + "constants.ncl") loadscript(lib_location + "strcmp_exact.ncl") loadscript(lib_location + "strpad.ncl") loadscript(lib_location + "repeat_str.ncl") +loadscript(lib_location + "interpol_extrapol_linear.ncl") loadscript(lib_location + "calc_rotated_sphr_coords_from_sphr.ncl") loadscript(lib_location + "calc_sphr_coords_from_rotated_sphr.ncl") loadscript(lib_location + "convert_from_to_sphr_coords_to_from_rotated_sphr.ncl") @@ -17,10 +18,13 @@ loadscript(lib_location + "convert_from_to_sphr_coords_to_from_rotated_sphr.ncl" undef("calc_wrtcmp_grid_params_rotated_latlon") function calc_wrtcmp_grid_params_rotated_latlon( \ - wrtcmp_config_fn:string, \ + model_config_tmpl_fp:string, \ lon_ctr_native:snumeric, lat_ctr_native:snumeric, \ lon_tile_corners_face_midpts_native[8]:snumeric, \ lat_tile_corners_face_midpts_native[8]:snumeric, \ + lon_bdy_native[*]:snumeric, \ + lat_bdy_native[*]:snumeric, \ + nx_native:snumeric, ny_native:snumeric, \ dx_native:snumeric, dy_native:snumeric, \ angle_units:string) @@ -113,8 +117,6 @@ print("(lon_tile_corners_face_midpts_native, lat_tile_corners_face_midpts_native pause end if - - ; ; ********************************************************************** ; @@ -123,6 +125,12 @@ end if ; ; ********************************************************************** ; +;print("") +;print("(lon_tile_corners_face_midpts_native, lat_tile_corners_face_midpts_native) = " + \ +; lon_tile_corners_face_midpts_native + ", " + \ +; lat_tile_corners_face_midpts_native) +;pause + rotated_sphr_coords \ := convert_from_to_sphr_coords_to_from_rotated_sphr( \ lon_ctr, lat_ctr, angle_units, 1, \ @@ -130,6 +138,15 @@ end if lat_tile_corners_face_midpts_native) x_tile_corners_face_midpts_native = rotated_sphr_coords@lon_out y_tile_corners_face_midpts_native = rotated_sphr_coords@lat_out + + rotated_sphr_coords \ + := convert_from_to_sphr_coords_to_from_rotated_sphr( \ + lon_ctr, lat_ctr, angle_units, 1, \ + lon_bdy_native, \ + lat_bdy_native) + x_bdy_native = rotated_sphr_coords@lon_out + y_bdy_native = rotated_sphr_coords@lat_out + ;print("") ;print("(x_tile_corners_face_midpts_native, y_tile_corners_face_midpts_native) = " + \ ; x_tile_corners_face_midpts_native + ", " + \ @@ -219,8 +236,103 @@ end if y_S_native_max = max((/y_SW_native, y_S_native, y_SE_native/)) y_N_native_min = min((/y_NW_native, y_N_native, y_NE_native/)) +; +; The following code is for the case in which in the xy-plane [i.e. in the +; lonp-latp plane, where lonp and latp are the rotated latlon coordinates +; (with the "p" indicating "prime")] the left and right boundaries are +; concave (bow into the domain) and the bottom and top boundaries are +; convex (bow out of the domain). It allows the write-component domain +; in the y (i.e. latp) direction to be slightly larger by calculating +; the lower and upper bounds of the domain in a less conservative way. +; This needs to be generalized to cases in which the left and right +; boundaries are convex and the bottom and top boundaries are convex. +; +; This code (after generalization) needs to be included in the file +; calc_wrtcmp_grid_params_lambert_cnfrml.ncl. +; +if (True) then +;if (False) then + + if ((y_S_native .lt. y_SW_native) .and. (y_S_native .lt. y_SE_native)) then + + indx_start = 0 + indx_end = indx_start + nx_native + x_bdy_native_bot := x_bdy_native(indx_start:indx_end) + y_bdy_native_bot := y_bdy_native(indx_start:indx_end) + + inds_sort := dim_pqsort(x_bdy_native_bot, 2) + y_bdy_native_bot := y_bdy_native_bot(inds_sort) + + indx_nearest = ind_nearest_coord(x_W_native_max, x_bdy_native_bot, 0) + if (x_W_native_max .lt. x_bdy_native_bot(indx_nearest)) then + indx_1 = indx_nearest - 1 + indx_2 = indx_nearest + else if (x_W_native_max .ge. x_bdy_native_bot(indx_nearest)) then + indx_1 = indx_nearest + indx_2 = indx_nearest + 1 + end if + end if + +; y_S_native_max_linear := interpol_extrapol_linear( \ +; x_SW_native, y_SW_native, \ +; x_S_native, y_S_native, \ +; x_W_native_max) + y_S_native_max = interpol_extrapol_linear( \ + x_bdy_native_bot(indx_1), y_bdy_native_bot(indx_1), \ + x_bdy_native_bot(indx_2), y_bdy_native_bot(indx_2), \ + x_W_native_max) +;print("NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN") +;print("y_S_native_max_linear = " + y_S_native_max_linear) +;print("y_S_native_max = " + y_S_native_max) +;pause + + end if + + + if ((y_N_native .gt. y_NW_native) .and. (y_N_native .gt. y_NE_native)) then + + indx_start = nx_native + ny_native + indx_end = indx_start + nx_native + x_bdy_native_top := x_bdy_native(indx_start:indx_end) + y_bdy_native_top := y_bdy_native(indx_start:indx_end) + + inds_sort := dim_pqsort(x_bdy_native_top, 2) + y_bdy_native_top := y_bdy_native_top(inds_sort) + + indx_nearest = ind_nearest_coord(x_W_native_max, x_bdy_native_top, 0) + if (x_W_native_max .lt. x_bdy_native_top(indx_nearest)) then + indx_1 = indx_nearest - 1 + indx_2 = indx_nearest + else if (x_W_native_max .ge. x_bdy_native_top(indx_nearest)) then + indx_1 = indx_nearest + indx_2 = indx_nearest + 1 + end if + end if + +; y_N_native_min_linear := interpol_extrapol_linear( \ +; x_NW_native, y_NW_native, \ +; x_N_native, y_N_native, \ +; x_W_native_max) + y_N_native_min = interpol_extrapol_linear( \ + x_bdy_native_top(indx_1), y_bdy_native_top(indx_1), \ + x_bdy_native_top(indx_2), y_bdy_native_top(indx_2), \ + x_W_native_max) +;print("NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN") +;print("y_N_native_min_linear = " + y_N_native_min_linear) +;print("y_N_native_min = " + y_N_native_min) +;pause + + end if + +end if + + + y_S = y_S_native_max + num_margin_cells*dy y_N = y_N_native_min - num_margin_cells*dy +print("") +print("y_S = " + y_S) +print("y_N = " + y_N) y_S = -min(abs((/y_S, y_N/))) y_N = -y_S @@ -236,7 +348,7 @@ end if ; ********************************************************************** ; ; Calculate the spherical coordinates of the southwest corner of the -; native grid from its Lambert coordinates. +; native grid from its rotated latlon coordinates. ; ; ********************************************************************** ; @@ -300,16 +412,16 @@ end if param_value := char_sq + "rotated_latlon" + char_sq param_value_str := tostring(param_value) - regex_search = "^(\s*" + param_name + ":\s+)(<" + param_name + ">)(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)('\{\{\s*" + param_name + "\s*\}\}')(\s*)(.*)" regex_print = "\1" + param_value_str sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp param_names_and_vals(np) = systemfunc(sed_cmd) regex_print = "\4" sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp trailing_comments(np) = systemfunc(sed_cmd) ; ; ********************************************************************** @@ -366,7 +478,7 @@ end if end if ; ; Generate a string containing the parameter value and formatted as spe- -; cified by fmt_str. Then strip any leading and trailing whitespace +; cified by fmt_str. Then strip any leading and trailing whitespace ; from it. ; param_value_str := sprintf(fmt_str, param_value) @@ -374,22 +486,24 @@ end if ; ; Set the regular expression to search for. ; - regex_search = "^(\s*" + param_name + ":\s+)(<" + param_name + ">)(\s*)(.*)" + regex_search = "^(\s*" + param_name + ":\s+)(\{\{\s*" + param_name + "\s*\}\})(\s*)(.*)" ; ; Get the parameter name and value without the trailing comment (if any). ; regex_print = "\1" + param_value_str sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn - param_names_and_vals(np) = systemfunc(sed_cmd) + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp + tmp := systemfunc(sed_cmd) + param_names_and_vals(np) = tmp(0) ; ; Get the trailing name and comment. ; regex_print = "\4" sed_cmd = "sed -n -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_print + "|p" + char_dq + " " + wrtcmp_config_fn - trailing_comments(np) = systemfunc(sed_cmd) - + + regex_print + "|p" + char_dq + " " + model_config_tmpl_fp + tmp := systemfunc(sed_cmd) + trailing_comments(np) = tmp(0) + end do ; ; ********************************************************************** @@ -429,7 +543,7 @@ end if regex_search = "^(\s*" + param_name + ":\s+)(<" + param_name + ">)(\s*)(.*)" regex_replace = lines_final(np) sed_cmd = "sed -i -r -e " + char_dq + "s|" + regex_search + "|" \ - + regex_replace + "|" + char_dq + " " + wrtcmp_config_fn + + regex_replace + "|" + char_dq + " " + model_config_tmpl_fp sed_output = systemfunc(sed_cmd) end do ; diff --git a/ush/NCL/find_wrtcmp_grid_params.ncl b/ush/NCL/find_wrtcmp_grid_params.ncl index fce1627f8e..01ab25f709 100644 --- a/ush/NCL/find_wrtcmp_grid_params.ncl +++ b/ush/NCL/find_wrtcmp_grid_params.ncl @@ -30,6 +30,9 @@ expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_ expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_feature_maximize_CONUS_grids/expt_dirs/try01/grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid01/grid_GSD_RAP13km_pregen" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid01/grid_GSD_RAP13km_pregen_old003" + ; ; ********************************************************************** ; @@ -202,7 +205,8 @@ print("AAAAAAAAAAAAAAAAAAAAAAAAAAA") ; print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") remove_rgnl_halo = True - get_tile_bdies = False +; get_tile_bdies = False + get_tile_bdies = True grid_info := read_FV3LAM_grid_native( \ expt_dir, \ @@ -221,6 +225,18 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") lat_tile_corners_face_midpts_tiles_to_plot \ = grid_info@lat_tile_corners_face_midpts_all_tiles + lon_bdy_tiles_to_plot = grid_info@lon_bdy_all_tiles + lat_bdy_tiles_to_plot = grid_info@lat_bdy_all_tiles + + nx_all_tiles := grid_info@nx_all_tiles + ny_all_tiles := grid_info@ny_all_tiles + +;print("LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLL") +;print("dimsizes(lon_tile_corners_face_midpts_tiles_to_plot) = " + dimsizes(lon_tile_corners_face_midpts_tiles_to_plot)) +;print("dimsizes(lon_bdy_tiles_to_plot) = " + dimsizes(lon_bdy_tiles_to_plot)) +;print("dimsizes(nx_all_tiles) = " + dimsizes(nx_all_tiles)) +;pause + ;lon_tile_corners_face_midpts_tiles_to_plot \ ;= where(lon_tile_corners_face_midpts_tiles_to_plot .gt. 0.0d+0, \ ; lon_tile_corners_face_midpts_tiles_to_plot - 360.0d+0, \ @@ -249,6 +265,19 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") lat_tile_corners_face_midpts_native \ := lat_tile_corners_face_midpts_tiles_to_plot(nn,:) + nx_native := nx_all_tiles(nn) + ny_native := ny_all_tiles(nn) +;print("nx_native = " + nx_native) +;print("ny_native = " + ny_native) +;pause + + indx_start := 0 + indx_end := 2*(nx_native + ny_native) + lon_bdy_native := lon_bdy_tiles_to_plot(indx_start:indx_end) + lat_bdy_native := lat_bdy_tiles_to_plot(indx_start:indx_end) +;print("dimsizes(lon_bdy_native) = " + dimsizes(lon_bdy_native)) +;pause + angle_units = "deg" valid_vals_WRTCMP_output_grid \ @@ -264,6 +293,9 @@ print("BBBBBBBBBBBBBBBBBBBBBBBBBBB") lon_grid_cntr_native, lat_grid_cntr_native, \ lon_tile_corners_face_midpts_native, \ lat_tile_corners_face_midpts_native, \ + lon_bdy_native, \ + lat_bdy_native, \ + nx_native, ny_native, \ dx_native, dy_native, \ angle_units) diff --git a/ush/NCL/plot_grid.ncl b/ush/NCL/plot_grid.ncl index d33466d46a..4b1874fd36 100644 --- a/ush/NCL/plot_grid.ncl +++ b/ush/NCL/plot_grid.ncl @@ -99,6 +99,11 @@ CDATE="2019070100" ;expt_dir = "/work/noaa/gsd-fv3-dev/gsketefia/UFS/PR_release_branch/expt_dirs/try02/grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2" ;CDATE="2019052000" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid01/grid_GSD_RAP13km_pregen" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid02/grid_GSD_RAP13km_pregen" +;expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid02/grid_GSD_RAP13km_pregen_old003" +expt_dir = "/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/PR_Jeff_Duda_subhourly_post/expt_dirs/big_grid02/grid_GSD_RAP13km" +CDATE="2019070100" run_dir = expt_dir + "/" + CDATE print("AAAAAA run_dir = " + run_dir) @@ -156,6 +161,8 @@ map_proj = "cyln" ; map_proj_ctr = (/ 0.0, 0.0 /) ; This should be set to (0,0) for "cyln" map projection. ;map_proj_ctr = (/ -106.0, 54.0 /) ; RAP domain. +;map_proj_ctr = (/ -106.0, 84.0 /) ; RAP domain. +;map_proj_ctr = (/ 30.0, 60.0 /) ; RAP domain. ;map_proj_ctr = (/ -97.5, 38.5 /) ; HRRR domain. ;map_proj_ctr = (/ 6.0, 54.0 /) ; RAP domain corner. @@ -167,14 +174,33 @@ map_proj_ctr = (/ 0.0, 0.0 /) ; This should be set to (0,0) for "cyln" map proje plot_subreg = True ;plot_subreg = False -;subreg_limits := (/ -180, 180, -90, 90 /) +subreg_limits := (/ -180, 180, -90, 90 /) ;subreg_limits := (/ -75, -45, 30, 60 /) - - - - -subreg_limits := (/ -128, -67, 23, 51 /) ; HRRR domain if using "cyln" map projection. -subreg_limits := (/ -135, -60, 20, 55 /) ; HRRR domain if using "cyln" map projection. +;subreg_limits := (/ -155, -45, 60, 90 /) +;subreg_limits := (/ -155, -45, 60, 100 /) +;subreg_limits := (/ -190, -170, 80, 100 /) +;subreg_limits := (/ -55, -45, 80, 90 /) +;subreg_limits := (/ 45, 55, 60, 70 /) +;subreg_limits := (/ 15, 45, 30, 50 /) + + +;subreg_limits := (/ -75, -45, -30, 0 /) ; +;subreg_limits := (/ -40, -35, 25, 30 /) ; + +; The following are to plot regions near the four corners and along two +; of the edges of GSD_RAP13km domain (soon to be renamed to RRFS_NA_13km) +; to ensure that the write-component grid is within the native grid. It's +; easiest to use map_proj = "cyln" to plot these subregions. +;subreg_limits := (/ -155, -130, -25, 0 /) ; +;subreg_limits := (/ -80, -55, -25, 0 /) ; +;subreg_limits := (/ -45, -20, 15, 40 /) +;subreg_limits := (/ 5, 30, 35, 60 /) +;subreg_limits := (/ 125, 150, 35, 60 /) +;subreg_limits := (/ -180, -155, 15, 40 /) + + +;subreg_limits := (/ -128, -67, 23, 51 /) ; HRRR domain if using "cyln" map projection. +;subreg_limits := (/ -135, -60, 20, 55 /) ; HRRR domain if using "cyln" map projection. ;subreg_limits := (/ -125, -70, 20, 55 /) ; HRRR domain if using "ortho" map projection. ;subreg_limits := (/ -105, -95, 20, 30 /) ; Center of southern edge of HRRR domain. @@ -659,7 +685,7 @@ wrtcmp@bdy_line_thickness = 6.0 wrtcmp@bdy_dash_pattern = 1 extrnl = True -;extrnl = False +extrnl = False ;extrnl@plot_gridlines = True extrnl@plot_gridlines = False ;extrnl@gridline_color = "red" diff --git a/ush/NCL/read_FV3LAM_grid_wrtcmp.ncl b/ush/NCL/read_FV3LAM_grid_wrtcmp.ncl index 7ed14cbe9d..6ed3001613 100644 --- a/ush/NCL/read_FV3LAM_grid_wrtcmp.ncl +++ b/ush/NCL/read_FV3LAM_grid_wrtcmp.ncl @@ -10,8 +10,8 @@ ; cell vertex coordinates of the grid that discretizes the FV3-LAM's ; write-component output domain. The file to read is specified by ; FV3LAM_wrtcmp_fn. If get_domain_bdy is set to True, this function -; also returns the coordinates of the cell vertices lying on the bounda- -; ry of the domain. +; also returns the coordinates of the cell vertices lying on the boundary +; of the domain. ; ; ********************************************************************** ; @@ -20,6 +20,7 @@ loadscript(lib_location + "constants.ncl") loadscript(lib_location + "strcmp_exact.ncl") loadscript(lib_location + "get_rect_grid_bdy.ncl") loadscript(lib_location + "interpol_extrapol_linear.ncl") +loadscript(lib_location + "convert_from_to_sphr_coords_to_from_rotated_sphr.ncl") undef("read_FV3LAM_grid_wrtcmp") @@ -88,7 +89,7 @@ begin ; ; ********************************************************************** ; -; Read in the cell vertex coordinates. +; Read in the cell center coordinates. ; ; ********************************************************************** ; @@ -121,6 +122,72 @@ begin print("") print(" nx = " + nx) print(" ny = " + ny) + +; +; ********************************************************************** +; +; If the grid encompasses the international date line (IDL, e.g. because +; it includes one of the poles), then the extrapolations performed below +; to obtain the cell vertex coordinates along the boundary of the grid +; may be invalid because the coordinates themselves are not continuous on +; the domain (because there will be a jump in longitude across the IDL). +; To get around this problem, transform the write-component coordinates +; read in above to a rotated latlon coordinate system that has is IDL +; moved such that it is outside of the write-component domain. In this +; coordinate system, the coordinates will be continuous and thus the +; extrapolations below will be valid. +; +; Once all quantities that this function needs to return are calculated +; in the rotated latlon coordinate system, they will be transformed back +; to obtain their values in the non-rotated coordinate system (see further +; below). +; +; IMPORTANT: +; Probably will have to include code similar to the following in the file +; read_FV3LAM_grid_native.ncl. +; +; ********************************************************************** +; + nxm1 := nx - 1 + nym1 := ny - 1 + nxp1 := nx + 1 + nyp1 := ny + 1 + + diffx_lon_iph := lon_cntrs(:,1:nxm1) - lon_cntrs(:,0:nxm1-1) + diffy_lon_jph := lon_cntrs(1:nym1,:) - lon_cntrs(0:nym1-1,:) + + diffx_lon_abs_max := max(abs(diffx_lon_iph)) + diffy_lon_abs_max := max(abs(diffy_lon_jph)) + diff_lon_abs_max := 180.0d+0 + + transform_to_rotated_latlon = False + if ((diffx_lon_abs_max .gt. diff_lon_abs_max) .or. \ + (diffy_lon_abs_max .gt. diff_lon_abs_max)) then +; Set the flag to perform the unrotated-to-rotated latlon transformation +; (and back again later on). + transform_to_rotated_latlon = True +; Read in from the specified file the coordinates of the center of the +; grid (in non-rotated latlon coordinates). + file_global_atts = getfileatts(fp) + lon0 := fp@cen_lon + lat0 := fp@cen_lat + end if +; +; If necessary, transform the grid cell center coordinates from the +; unrotated to a rotated latlon coordinate system. +; + if (transform_to_rotated_latlon) then + + angle_units = "deg" + dir = 1 + rotated_sphr_coords \ + := convert_from_to_sphr_coords_to_from_rotated_sphr( \ + lon0, lat0, angle_units, dir, \ + lon_cntrs, lat_cntrs) + lon_cntrs := rotated_sphr_coords@lon_out + lat_cntrs := rotated_sphr_coords@lat_out + + end if ; ; ********************************************************************** ; @@ -134,8 +201,8 @@ begin ; ; ********************************************************************** ; - lon_verts := new((/ ny+1, nx+1 /), coord_data_type) - lat_verts := new((/ ny+1, nx+1 /), coord_data_type) + lon_verts := new((/ nyp1, nxp1 /), coord_data_type) + lat_verts := new((/ nyp1, nxp1 /), coord_data_type) ;lon_verts = -98.5d+0 ;lat_verts = 37.5d+0 ; @@ -143,24 +210,24 @@ begin ; each vertex that is in the interior of the domain/grid to obtain the ; coordinates of that internal vertex. ; - lon_verts(1:ny-1,1:nx-1) \ - = (lon_cntrs(0:ny-2,0:nx-2) \ - + lon_cntrs(0:ny-2,1:nx-1) \ - + lon_cntrs(1:ny-1,1:nx-1) \ - + lon_cntrs(1:ny-1,0:nx-2))/4.0d+0 + lon_verts(1:nym1,1:nxm1) \ + = (lon_cntrs(0:nym1-1,0:nxm1-1) \ + + lon_cntrs(0:nym1-1,1:nxm1) \ + + lon_cntrs(1:nym1,1:nxm1) \ + + lon_cntrs(1:nym1,0:nxm1-1))/4.0d+0 - lat_verts(1:ny-1,1:nx-1) \ - = (lat_cntrs(0:ny-2,0:nx-2) \ - + lat_cntrs(0:ny-2,1:nx-1) \ - + lat_cntrs(1:ny-1,1:nx-1) \ - + lat_cntrs(1:ny-1,0:nx-2))/4.0d+0 + lat_verts(1:nym1,1:nxm1) \ + = (lat_cntrs(0:nym1-1,0:nxm1-1) \ + + lat_cntrs(0:nym1-1,1:nxm1) \ + + lat_cntrs(1:nym1,1:nxm1) \ + + lat_cntrs(1:nym1,0:nxm1-1))/4.0d+0 ; ; Create integer (index) arrays needed in the extrapolations below in the ; x direction. ; - ivec1 := new(ny-1, "integer") - ivec2 := new(ny-1, "integer") - ivec3 := new(ny-1, "integer") + ivec1 := new(nym1, "integer") + ivec2 := new(nym1, "integer") + ivec3 := new(nym1, "integer") ; ; Extrapolate to obtain coordinates of vertices along the left boundary ; (but not including the vertices on the bottom-left and top-left corners @@ -172,12 +239,12 @@ begin ivec1 = i1 ivec2 = i2 ivec3 = i3 - lon_verts(1:ny-1,i3) \ + lon_verts(1:nym1,i3) \ = interpol_extrapol_linear( \ - ivec1, lon_verts(1:ny-1,i1), ivec2, lon_verts(1:ny-1,i2), ivec3) - lat_verts(1:ny-1,i3) \ + ivec1, lon_verts(1:nym1,i1), ivec2, lon_verts(1:nym1,i2), ivec3) + lat_verts(1:nym1,i3) \ = interpol_extrapol_linear( \ - ivec1, lat_verts(1:ny-1,i1), ivec2, lat_verts(1:ny-1,i2), ivec3) + ivec1, lat_verts(1:nym1,i1), ivec2, lat_verts(1:nym1,i2), ivec3) ; ; Extrapolate to obtain coordinates of vertices along the right boundary ; (but not including the vertices on the bottom-right and top-right corners @@ -189,19 +256,19 @@ begin ivec1 = i1 ivec2 = i2 ivec3 = i3 - lon_verts(1:ny-1,i3) \ + lon_verts(1:nym1,i3) \ = interpol_extrapol_linear( \ - ivec1, lon_verts(1:ny-1,i1), ivec2, lon_verts(1:ny-1,i2), ivec3) - lat_verts(1:ny-1,i3) \ + ivec1, lon_verts(1:nym1,i1), ivec2, lon_verts(1:nym1,i2), ivec3) + lat_verts(1:nym1,i3) \ = interpol_extrapol_linear( \ - ivec1, lat_verts(1:ny-1,i1), ivec2, lat_verts(1:ny-1,i2), ivec3) + ivec1, lat_verts(1:nym1,i1), ivec2, lat_verts(1:nym1,i2), ivec3) ; ; Create integer (index) arrays needed in the extrapolations below in the ; y direction. ; - jvec1 := new(nx-1, "integer") - jvec2 := new(nx-1, "integer") - jvec3 := new(nx-1, "integer") + jvec1 := new(nxm1, "integer") + jvec2 := new(nxm1, "integer") + jvec3 := new(nxm1, "integer") ; ; Extrapolate to obtain coordinates of vertices along the bottom boundary ; (but not including the vertices on the bottom-left and bottom-right @@ -213,12 +280,12 @@ begin jvec1 = j1 jvec2 = j2 jvec3 = j3 - lon_verts(j3,1:nx-1) \ + lon_verts(j3,1:nxm1) \ = interpol_extrapol_linear( \ - jvec1, lon_verts(j1,1:nx-1), jvec2, lon_verts(j2,1:nx-1), jvec3) - lat_verts(j3,1:nx-1) \ + jvec1, lon_verts(j1,1:nxm1), jvec2, lon_verts(j2,1:nxm1), jvec3) + lat_verts(j3,1:nxm1) \ = interpol_extrapol_linear( \ - jvec1, lat_verts(j1,1:nx-1), jvec2, lat_verts(j2,1:nx-1), jvec3) + jvec1, lat_verts(j1,1:nxm1), jvec2, lat_verts(j2,1:nxm1), jvec3) ; ; Extrapolate to obtain coordinates of vertices along the top boundary ; (but not including the vertices on the top-left and top-right corners @@ -230,12 +297,12 @@ begin jvec1 = j1 jvec2 = j2 jvec3 = j3 - lon_verts(j3,1:nx-1) \ + lon_verts(j3,1:nxm1) \ = interpol_extrapol_linear( \ - jvec1, lon_verts(j1,1:nx-1), jvec2, lon_verts(j2,1:nx-1), jvec3) - lat_verts(j3,1:nx-1) \ + jvec1, lon_verts(j1,1:nxm1), jvec2, lon_verts(j2,1:nxm1), jvec3) + lat_verts(j3,1:nxm1) \ = interpol_extrapol_linear( \ - jvec1, lat_verts(j1,1:nx-1), jvec2, lat_verts(j2,1:nx-1), jvec3) + jvec1, lat_verts(j1,1:nxm1), jvec2, lat_verts(j2,1:nxm1), jvec3) ; ; ********************************************************************** ; @@ -356,17 +423,17 @@ begin ; ********************************************************************** ; lon_verts_unstruc \ - := (/ ndtooned(lon_verts(0:ny-1,0:nx-1)), \ - ndtooned(lon_verts(0:ny-1,1:nx)), \ + := (/ ndtooned(lon_verts(0:nym1,0:nxm1)), \ + ndtooned(lon_verts(0:nym1,1:nx)), \ ndtooned(lon_verts(1:ny,1:nx)), \ - ndtooned(lon_verts(1:ny,0:nx-1)) /) + ndtooned(lon_verts(1:ny,0:nxm1)) /) lon_verts_unstruc := transpose(lon_verts_unstruc) lat_verts_unstruc \ - := (/ ndtooned(lat_verts(0:ny-1,0:nx-1)), \ - ndtooned(lat_verts(0:ny-1,1:nx)), \ + := (/ ndtooned(lat_verts(0:nym1,0:nxm1)), \ + ndtooned(lat_verts(0:nym1,1:nx)), \ ndtooned(lat_verts(1:ny,1:nx)), \ - ndtooned(lat_verts(1:ny,0:nx-1)) /) + ndtooned(lat_verts(1:ny,0:nxm1)) /) lat_verts_unstruc := transpose(lat_verts_unstruc) ; ; ********************************************************************** @@ -418,6 +485,42 @@ begin ; ; ********************************************************************** ; +; If the original latlon grid coordinates read in from the file were +; transformed above to a rotated coordinate system because they include +; the international date line, then transform the results obtained above +; back to the non-rotated latlon coordinate system before returning them. +; +; ********************************************************************** +; + if (transform_to_rotated_latlon) then + + dir = -1 + + sphr_coords \ + := convert_from_to_sphr_coords_to_from_rotated_sphr( \ + lon0, lat0, angle_units, dir, \ + lon_cntrs_unstruc, lat_cntrs_unstruc) + lon_cntrs_unstruc := sphr_coords@lon_out + lat_cntrs_unstruc := sphr_coords@lat_out + + sphr_coords \ + := convert_from_to_sphr_coords_to_from_rotated_sphr( \ + lon0, lat0, angle_units, dir, \ + lon_verts_unstruc, lat_verts_unstruc) + lon_verts_unstruc := sphr_coords@lon_out + lat_verts_unstruc := sphr_coords@lat_out + + sphr_coords \ + := convert_from_to_sphr_coords_to_from_rotated_sphr( \ + lon0, lat0, angle_units, dir, \ + lon_bdy, lat_bdy) + lon_bdy := sphr_coords@lon_out + lat_bdy := sphr_coords@lat_out + + end if +; +; ********************************************************************** +; ; Return results as attributes of the logical variable grid_info. ; ; ********************************************************************** diff --git a/ush/set_gridparams_ESGgrid.sh b/ush/set_gridparams_ESGgrid.sh index 8cef16ee36..3c0fa0f92a 100644 --- a/ush/set_gridparams_ESGgrid.sh +++ b/ush/set_gridparams_ESGgrid.sh @@ -80,8 +80,7 @@ function set_gridparams_ESGgrid() { # #----------------------------------------------------------------------- # -# Source the file containing various mathematical, physical, etc cons- -# tants. +# Source the file containing various mathematical, physical, etc constants. # #----------------------------------------------------------------------- # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index f01ab6d306..cfb4136ba1 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -10,6 +10,15 @@ function set_predef_grid_params() { # #----------------------------------------------------------------------- # +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# # Get the full path to the file in which this script/function is located # (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in # which the file is located (scrfunc_dir). @@ -30,6 +39,14 @@ local func_name="${FUNCNAME[0]}" # #----------------------------------------------------------------------- # +# Source the file containing various mathematical, physical, etc constants. +# +#----------------------------------------------------------------------- +# +. ${USHDIR}/constants.sh +# +#----------------------------------------------------------------------- +# # Set grid and other parameters according to the value of the predefined # domain (PREDEF_GRID_NAME). Note that the code will enter this script # only if PREDEF_GRID_NAME has a valid (and non-empty) value. @@ -317,8 +334,10 @@ case ${PREDEF_GRID_NAME} in # WRTCMP_lat_lwr_left="-13.56176982" # WRTCMP_lon_upr_rght="18.47206579" # WRTCMP_lat_upr_rght="13.56176982" -# WRTCMP_dlon="0.11691181" -# WRTCMP_dlat="0.11691181" +## WRTCMP_dlon="0.11691181" +## WRTCMP_dlat="0.11691181" +# WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) +# WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) # fi ;; # @@ -1068,7 +1087,7 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"GSD_RAP13km") +"RRFS_NA_13km") # if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # @@ -1139,16 +1158,25 @@ case ${PREDEF_GRID_NAME} in WRTCMP_output_grid="rotated_latlon" WRTCMP_cen_lon="${ESGgrid_LON_CTR}" WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-57.9926" - WRTCMP_lat_lwr_left="-50.74344" - WRTCMP_lon_upr_rght="57.99249" - WRTCMP_lat_upr_rght="50.74344" - WRTCMP_dlon="0.1218331" - WRTCMP_dlat="0.121833" + WRTCMP_lon_lwr_left="-55.82538869" + WRTCMP_lat_lwr_left="-48.57685654" + WRTCMP_lon_upr_rght="55.82538869" + WRTCMP_lat_upr_rght="48.57685654" + WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) + WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi ;; esac +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 } # diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index f9f4f7c6d7..eb74d0f0bf 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -19,7 +19,7 @@ valid_vals_PREDEF_GRID_NAME=( \ "GSL_HAFSV0.A_13km" \ "GSL_HAFSV0.A_3km" \ "GSD_HRRR_AK_50km" \ -"GSD_RAP13km" \ +"RRFS_NA_13km" \ ) valid_vals_CCPP_PHYS_SUITE=( \ "FV3_CPT_v0" \ From c53342cc92e967be0e6927ff47ee1cf1f7736b5d Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Mon, 5 Apr 2021 07:22:37 -0600 Subject: [PATCH 028/203] Fixed a couple typos and modified syntax for status message to work correctly (#463) --- scripts/exregional_get_mrms_files.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index 01d95043d7..ce6117fb4d 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -70,17 +70,17 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do # Name of MRMS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20200303 ]]; then - CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" >& /dev/null &` + CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" >& /dev/null` Status=$? if [[ ${Status} == 0 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" else - CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" >& /dev/null &` + CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" >& /dev/null` Status=$? if [[ ${Status} == 0 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" + TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" else - echo "ERROR: MRMR data not available for ${vyyyy}${vmm}${vdd}!" + echo "ERROR: MRMS data not available for ${vyyyy}${vmm}${vdd}!" exit fi fi From 1df37a33e1fc5de6ff022e5465b6bc2fad1d9d4a Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 7 Apr 2021 13:55:38 -0600 Subject: [PATCH 029/203] Add functionality for user-defined OMP settings and for the RUN_FCST "--cpus-per-task" value. (#459) * Create user-defined OMP variables. * Add OMP variables to make_sfc_climo task * Move OMP stanza in exregional_run_fcst.sh file * Add "OpenMP" to comment. * Make --cpus-per-task a user-defined variable. * Revert to "srun" for the RUN_FCST task on Jet * Remove export commands from ex-scripts, remove OMP options from make_grid and make_orog tasks. * Add OMP options to make_orog ex-script. * Remove KMP_AFFINITY setting from WCOSS_CRAY block. * Remove exports from all commands in exregional_make_orog.sh and update comments in all relevant ex-scripts. --- scripts/exregional_make_grid.sh | 10 ----- scripts/exregional_make_ics.sh | 10 +++++ scripts/exregional_make_lbcs.sh | 10 +++++ scripts/exregional_make_orog.sh | 20 ++++----- scripts/exregional_make_sfc_climo.sh | 10 +++++ scripts/exregional_run_fcst.sh | 25 +++++------ scripts/exregional_run_post.sh | 10 +++++ ush/config_defaults.sh | 65 +++++++++++++++++++++++++++- ush/generate_FV3LAM_wflow.sh | 2 +- 9 files changed, 126 insertions(+), 36 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index a7921b5bbf..163515ac9e 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -80,16 +80,6 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -# The orography code runs with threads. On Cray, the code is optimized -# for six threads. Do not change. -# -#----------------------------------------------------------------------- -# -export OMP_NUM_THREADS=6 -export OMP_STACKSIZE=2048m -# -#----------------------------------------------------------------------- -# # Set the machine-dependent run command. Also, set resource limits as # necessary. # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 3431b8b25d..d3d4617491 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -72,6 +72,16 @@ print_input_args valid_args # #----------------------------------------------------------------------- # +# Set OpenMP variables. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY=${KMP_AFFINITY_MAKE_ICS} +OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_ICS} +OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} +# +#----------------------------------------------------------------------- +# # Set machine-dependent parameters. # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index c847d11a84..839854fd6c 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -72,6 +72,16 @@ print_input_args valid_args # #----------------------------------------------------------------------- # +# Set OpenMP variables. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY=${KMP_AFFINITY_MAKE_LBCS} +OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_LBCS} +OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} +# +#----------------------------------------------------------------------- +# # Set machine-dependent parameters. # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 04f4dd0bc3..1687c42e39 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -76,13 +76,14 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -# The orography code runs with threads. On Cray, the code is optimized -# for six threads. Do not change. +# Set OpenMP variables. The orog executable runs with OMP. On +# WCOSS (Cray), it is optimized for six threads, which is the default. # #----------------------------------------------------------------------- # -export OMP_NUM_THREADS=6 -export OMP_STACKSIZE=2048m +KMP_AFFINITY=${KMP_AFFINITY_MAKE_OROG} +OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_OROG} +OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # @@ -102,9 +103,8 @@ case $MACHINE in module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 module list { restore_shell_opts; } > /dev/null 2>&1 - export NODES=1 - export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" - export KMP_AFFINITY=disabled + NODES=1 + APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" ulimit -s unlimited ulimit -a ;; @@ -130,11 +130,11 @@ case $MACHINE in "JET") ulimit -s unlimited ulimit -a - export APRUN="time" + APRUN="time" ;; "ODIN") - export APRUN="srun -n 1" + APRUN="srun -n 1" ulimit -s unlimited ulimit -a ;; @@ -144,7 +144,7 @@ case $MACHINE in ;; "STAMPEDE") - export APRUN="time" + APRUN="time" ;; *) diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 5d5c526d85..4880a35086 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -78,6 +78,16 @@ print_input_args valid_args # #----------------------------------------------------------------------- # +# Set OpenMP variables. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY=${KMP_AFFINITY_MAKE_SFC_CLIMO} +OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_SFC_CLIMO} +OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_SFC_CLIMO} +# +#----------------------------------------------------------------------- +# # Are these machine dependent?? # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index f59b0d3b3a..264957b533 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -83,6 +83,16 @@ print_input_args valid_args # #----------------------------------------------------------------------- # +# Set OpenMP variables. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY=${KMP_AFFINITY_RUN_FCST} +OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_FCST} +OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} +# +#----------------------------------------------------------------------- +# # Load modules. # #----------------------------------------------------------------------- @@ -110,7 +120,6 @@ case $MACHINE in ulimit -s unlimited ulimit -a APRUN="srun" - OMP_NUM_THREADS=4 ;; "ORION") @@ -123,7 +132,6 @@ case $MACHINE in ulimit -s unlimited ulimit -a APRUN="srun" - OMP_NUM_THREADS=4 ;; "ODIN") @@ -450,7 +458,7 @@ fi # create_model_configure_file \ cdate="$cdate" \ - nthreads=${OMP_NUM_THREADS:-1} \ + nthreads=${OMP_NUM_THREADS} \ run_dir="${run_dir}" \ sub_hourly_post="${SUB_HOURLY_POST}" \ dt_subhourly_post_mnts="${DT_SUBHOURLY_POST_MNTS}" \ @@ -479,17 +487,6 @@ if [ "${DO_ENSEMBLE}" = "TRUE" ]; then diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" ln_vrfy -sf ${relative_or_null} ${diag_table_fp} ${run_dir} fi -# -#----------------------------------------------------------------------- -# -# Set and export variables. -# -#----------------------------------------------------------------------- -# -export KMP_AFFINITY=scatter -export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} #Needs to be 1 for dynamic build of CCPP with GFDL fast physics, was 2 before. -export OMP_STACKSIZE=1024m - # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 1cddd61cc8..0fd169953c 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -78,6 +78,16 @@ print_input_args valid_args # #----------------------------------------------------------------------- # +# Set OpenMP variables. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY=${KMP_AFFINITY_RUN_POST} +OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_POST} +OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} +# +#----------------------------------------------------------------------- +# # Load modules. # #----------------------------------------------------------------------- diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index f0336561d7..336f00b6af 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1524,7 +1524,7 @@ SPP_STDDEV_CUTOFF=( "1.5" ) # #----------------------------------------------------------------------- # -HALO_BLEND=10 +HALO_BLEND="10" # #----------------------------------------------------------------------- # @@ -1579,4 +1579,67 @@ COMPILER="intel" #----------------------------------------------------------------------- # GWD_HRRRsuite_BASEDIR="" +# +#----------------------------------------------------------------------- +# +# KMP_AFFINITY_*: +# From Intel: "The Intel® runtime library has the ability to bind OpenMP +# threads to physical processing units. The interface is controlled using +# the KMP_AFFINITY environment variable. Depending on the system (machine) +# topology, application, and operating system, thread affinity can have a +# dramatic effect on the application speed. +# +# Thread affinity restricts execution of certain threads (virtual execution +# units) to a subset of the physical processing units in a multiprocessor +# computer. Depending upon the topology of the machine, thread affinity can +# have a dramatic effect on the execution speed of a program." +# +# For more information, see the following link: +# https://software.intel.com/content/www/us/en/develop/documentation/cpp- +# compiler-developer-guide-and-reference/top/optimization-and-programming- +# guide/openmp-support/openmp-library-support/thread-affinity-interface- +# linux-and-windows.html +# +# OMP_NUM_THREADS_*: +# The number of OpenMP threads to use for parallel regions. +# +# OMP_STACKSIZE_*: +# Controls the size of the stack for threads created by the OpenMP +# implementation. +# +# CPUS_PER_TASK_RUN_FCST: +# Sets the number of MPI tasks per CPU for the RUN_FCST task. +# +# Note that settings for the make_grid and make_orog tasks are not +# included below as they do not use parallelized code. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY_MAKE_OROG="scatter" +OMP_NUM_THREADS_MAKE_OROG="6" +OMP_STACKSIZE_MAKE_OROG="2048m" + +KMP_AFFINITY_MAKE_SFC_CLIMO="scatter" +OMP_NUM_THREADS_MAKE_SFC_CLIMO="1" +OMP_STACKSIZE_MAKE_SFC_CLIMO="1024m" + +KMP_AFFINITY_MAKE_ICS="scatter" +OMP_NUM_THREADS_MAKE_ICS="1" +OMP_STACKSIZE_MAKE_ICS="1024m" + +KMP_AFFINITY_MAKE_LBCS="scatter" +OMP_NUM_THREADS_MAKE_LBCS="1" +OMP_STACKSIZE_MAKE_LBCS="1024m" +KMP_AFFINITY_RUN_FCST="scatter" +OMP_NUM_THREADS_RUN_FCST="4" +OMP_STACKSIZE_RUN_FCST="1024m" + +CPUS_PER_TASK_RUN_FCST="4" + +KMP_AFFINITY_RUN_POST="scatter" +OMP_NUM_THREADS_RUN_POST="1" +OMP_STACKSIZE_RUN_POST="1024m" +# +#----------------------------------------------------------------------- +# diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 84c9c1fa7f..c7de625748 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -219,7 +219,7 @@ settings="\ # Number of cores used for a task # 'ncores_run_fcst': ${PE_MEMBER01} - 'native_run_fcst': --cpus-per-task 4 --exclusive + 'native_run_fcst': --cpus-per-task ${CPUS_PER_TASK_RUN_FCST} --exclusive # # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. From a546881f28c1fb4e71cf560f3e46e078cc75ed3d Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 8 Apr 2021 17:34:42 -0400 Subject: [PATCH 030/203] Modify path to external GFS data (#461) * Modify path to external GFS data * Modify path to external GFS data for WE2E --- ...ONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 3 +++ tests/run_experiments.sh | 2 +- ush/config.community.sh | 11 ++++++----- ush/config_defaults.sh | 2 +- ush/get_extrn_mdl_file_dir_info.sh | 6 +++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 35adb5d945..3a7d4f32fe 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -15,3 +15,6 @@ CYCL_HRS=( "18" ) EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" + +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 00ceb63ca1..f6d014060e 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -692,7 +692,7 @@ FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" elif [ "$MACHINE" = "HERA" ]; then - COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" + COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" elif [ "$MACHINE" = "JET" ]; then COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" elif [ "$MACHINE" = "CHEYENNE" ]; then diff --git a/ush/config.community.sh b/ush/config.community.sh index a1000ed905..8bbe0587ed 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -51,9 +51,10 @@ RUN_TASK_VX_POINTSTAT="FALSE" # # The following is specifically for Hera. It will have to be modified # if on another platform, using other dates, other external models, etc. +# Uncomment the following EXTRN_MDL_*_ICS/LBCS only when USE_USER_STAGED_EXTRN_FILES=TRUE # -EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" -EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/path/to/model_data/FV3GFS" -EXTRN_MDL_FILES_LBCS=( "gfs.pgrb2.0p25.f006" "gfs.pgrb2.0p25.f012" "gfs.pgrb2.0p25.f018" "gfs.pgrb2.0p25.f024" \ - "gfs.pgrb2.0p25.f030" "gfs.pgrb2.0p25.f036" "gfs.pgrb2.0p25.f042" "gfs.pgrb2.0p25.f048" ) +#EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" +#EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) +#EXTRN_MDL_SOURCE_BASEDIR_LBCS="/path/to/model_data/FV3GFS" +#EXTRN_MDL_FILES_LBCS=( "gfs.pgrb2.0p25.f006" "gfs.pgrb2.0p25.f012" "gfs.pgrb2.0p25.f018" "gfs.pgrb2.0p25.f024" \ +# "gfs.pgrb2.0p25.f030" "gfs.pgrb2.0p25.f036" "gfs.pgrb2.0p25.f042" "gfs.pgrb2.0p25.f048" ) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 336f00b6af..de56fd3e4f 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -173,7 +173,7 @@ EXPT_SUBDIR="" # (consisting of the 2-digit hour-of-day), the directory in which the # workflow will look for the external model files is: # -# $COMINgfs/gfs.$yyyymmdd/$hh +# $COMINgfs/gfs.$yyyymmdd/$hh/atmos # # FIXLAM_NCO_BASEDIR: # The base directory containing pregenerated grid, orography, and surface diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 56c7cf54b8..0de0a0c89f 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -675,13 +675,13 @@ has not been specified for this external model and machine combination: "FV3GFS") case "$MACHINE" in "WCOSS_CRAY") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" ;; "WCOSS_DELL_P3") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" ;; "HERA") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" ;; "ORION") sysdir="$sysbasedir" From d8266798d942fc2580c140f9e571a89abe0237ae Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 14 Apr 2021 14:23:42 -0600 Subject: [PATCH 031/203] =?UTF-8?q?Export=20OMP=20settings=20in=20the=20ma?= =?UTF-8?q?ke=5Forog=20ex-script=20and=20change=20the=20default=E2=80=A6?= =?UTF-8?q?=20(#474)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Export OMP settings in the make_orog ex-script and change the default KMP_AFFINITY for the make_orog task to match WCOSS. * Add export command to all OMP settings in each ex-script. --- scripts/exregional_make_ics.sh | 6 +++--- scripts/exregional_make_lbcs.sh | 6 +++--- scripts/exregional_make_orog.sh | 6 +++--- scripts/exregional_make_sfc_climo.sh | 6 +++--- scripts/exregional_run_fcst.sh | 6 +++--- scripts/exregional_run_post.sh | 6 +++--- ush/config_defaults.sh | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index d3d4617491..39ea2d90c0 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -76,9 +76,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_MAKE_ICS} -OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_ICS} -OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} +export KMP_AFFINITY=${KMP_AFFINITY_MAKE_ICS} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_ICS} +export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 839854fd6c..bdbe28cce6 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -76,9 +76,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_MAKE_LBCS} -OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_LBCS} -OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} +export KMP_AFFINITY=${KMP_AFFINITY_MAKE_LBCS} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_LBCS} +export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 1687c42e39..523d3b5cad 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -81,9 +81,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_MAKE_OROG} -OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_OROG} -OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} +export KMP_AFFINITY=${KMP_AFFINITY_MAKE_OROG} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_OROG} +export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 4880a35086..46fed72eeb 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -82,9 +82,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_MAKE_SFC_CLIMO} -OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_SFC_CLIMO} -OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_SFC_CLIMO} +export KMP_AFFINITY=${KMP_AFFINITY_MAKE_SFC_CLIMO} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_MAKE_SFC_CLIMO} +export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_SFC_CLIMO} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 264957b533..268639900e 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -87,9 +87,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_RUN_FCST} -OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_FCST} -OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} +export KMP_AFFINITY=${KMP_AFFINITY_RUN_FCST} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_FCST} +export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 0fd169953c..83b24f3a1f 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -82,9 +82,9 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -KMP_AFFINITY=${KMP_AFFINITY_RUN_POST} -OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_POST} -OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} +export KMP_AFFINITY=${KMP_AFFINITY_RUN_POST} +export OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_POST} +export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index de56fd3e4f..eb74163bae 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1615,7 +1615,7 @@ GWD_HRRRsuite_BASEDIR="" # #----------------------------------------------------------------------- # -KMP_AFFINITY_MAKE_OROG="scatter" +KMP_AFFINITY_MAKE_OROG="disabled" OMP_NUM_THREADS_MAKE_OROG="6" OMP_STACKSIZE_MAKE_OROG="2048m" From 87460c0eb8b894d6635d3335c9ed3f83152a04fc Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 16 Apr 2021 11:46:01 -0400 Subject: [PATCH 032/203] Add last three year global co2 data (#477) --- ush/config_defaults.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index eb74163bae..aaf38248d8 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1251,6 +1251,9 @@ FIXgsm_FILES_TO_COPY_TO_FIXam=( \ "fix_co2_proj/global_co2historicaldata_2016.txt" \ "fix_co2_proj/global_co2historicaldata_2017.txt" \ "fix_co2_proj/global_co2historicaldata_2018.txt" \ +"fix_co2_proj/global_co2historicaldata_2019.txt" \ +"fix_co2_proj/global_co2historicaldata_2020.txt" \ +"fix_co2_proj/global_co2historicaldata_2021.txt" \ "global_co2historicaldata_glob.txt" \ "co2monthlycyc.txt" \ "global_h2o_pltc.f77" \ @@ -1300,6 +1303,9 @@ CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING=( \ "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt" \ "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt" \ "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt" \ +"co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt" \ +"co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt" \ +"co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt" \ "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt" \ "co2monthlycyc.txt | co2monthlycyc.txt" \ "global_h2oprdlos.f77 | global_h2o_pltc.f77" \ From 791fad2da57c7eed0ea35edfc673ee8ecf339023 Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Mon, 19 Apr 2021 08:53:08 -0600 Subject: [PATCH 033/203] Update PR tempate to request name of branch PR will be issued to and documentation. (#478) --- .github/PULL_REQUEST_TEMPLATE | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 11192bdac4..a03ce9f331 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -4,7 +4,7 @@ - If you are unclear on what should be written here, see https://github.com/wrf-model/WRF/wiki/Making-a-good-pull-request-message for some guidance. -- The title of this pull request should be a brief summary (ideally less than 100 characters) of the changes included in this PR. +- The title of this pull request should be a brief summary (ideally less than 100 characters) of the changes included in this PR. Please also include the branch to which this PR is being issued. - Use the "Preview" tab to see what your PR will look like when you hit "Create pull request" @@ -16,6 +16,9 @@ One or more paragraphs describing the problem, solution, and required changes. ## TESTS CONDUCTED: Explicitly state what tests were run on these changes, or if any are still pending (for README or other text-only changes, just put "None required". Make note of the compilers used, the platform/machine, and other relevant details as necessary. For more complicated changes, or those resulting in scientific changes, please be explicit! +## DOCUMENTATION: +If this PR is contributing new capabilities that need to be documented, please also include updates to the RST files in the ufs-srweather-app repository (docs/UsersGuide/source) as supporting material. + ## ISSUE (optional): If this PR is resolving or referencing one or more issues, in this repository or elewhere, list them here. For example, "Fixes issue mentioned in #123" or "Related to bug in https://github.com/NOAA-EMC/other_repository/pull/63" From bd47867b47d3dba70b83e6b19d6f7ea41b47aece Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 21 Apr 2021 17:47:44 -0600 Subject: [PATCH 034/203] develop: Changes to processing and output related to sub-hourly simulation settings (#479) * Changes from Gerard's branch related to sub-hourly runs. Modifications and fixes added to setup.sh, the post ex-script, and the XML template as well. * Add subhourly WE2E tests * Remove empty lines caused by Jinja by adding dash to if statements. * Add subhourly minutes to valid values file and fix subhourly remainder calculation in setup.sh * Change DT_ATMOS to a value that will work with sub-hourly tests * Modify handling of "fmn" variable to work with both sub-hourly and hourly runs * Remove print statement/exit and changes to fill_jinja_template Python script (aside from typos). --- jobs/JREGIONAL_RUN_FCST | 1 - jobs/JREGIONAL_RUN_POST | 56 +++---- scripts/exregional_run_post.sh | 118 ++++++++------ ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 4 +- ...5km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 4 +- tests/baseline_configs/config.nco_ensemble.sh | 2 +- .../baseline_configs/config.subhourly_post.sh | 22 +++ .../config.subhourly_post_ensemble_2mems.sh | 25 +++ tests/baselines_list.txt | 2 + ush/config_defaults.sh | 38 +++-- ush/create_model_configure_file.sh | 46 ++++-- ush/fill_jinja_template.py | 4 +- ush/setup.sh | 138 +++++++++++----- ush/templates/FV3LAM_wflow.xml | 150 ++++++++++-------- ush/valid_param_vals.sh | 1 + 15 files changed, 396 insertions(+), 215 deletions(-) create mode 100644 tests/baseline_configs/config.subhourly_post.sh create mode 100644 tests/baseline_configs/config.subhourly_post_ensemble_2mems.sh diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 538b4a83c6..a4806c765e 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -73,7 +73,6 @@ mkdir_vrfy -p ${run_dir}/RESTART # #----------------------------------------------------------------------- # - $SCRIPTSDIR/exregional_run_fcst.sh \ cdate="${CDATE}" \ cycle_dir="${CYCLE_DIR}" \ diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index f54667ad4b..fca5ae21a4 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -63,20 +63,29 @@ on the output files corresponding to a specified forecast hour. # run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}" # +# If SUB_HOURLY_POST is not set to "TRUE", ensure that the forecast +# minutes (fmn) are set to "00". This is necessary in order to pass +# "fmn" into the post ex-script for the calculation of post_time. +# +if [ "${SUB_HOURLY_POST}" != "TRUE" ]; then + fmn="00" +fi +# #----------------------------------------------------------------------- # # If it doesn't already exist, create the directory (postprd_dir) in which -# to store post-processing output. (Note that postprd_dir may already -# have been created by this post-processing script run for a different -# forecast hour of the same cycle and/or ensemble member.) Also, create -# a temporary work directory (fhr_dir) for the current forecast hour. -# fhr_dir will be deleted later after the processing for the current -# forecast hour is complete. Then change location to fhr_dir. +# to store post-processing output. (Note that postprd_dir may have already +# been created by this post-processing script for a different output time +# from the same forecast start time and/or ensemble member.) Also, create +# a temporary work directory (tmp_dir) for the current output time. This +# will be deleted later after the processing for the current output time +# is complete. Then change location to tmp_dir. # -# Note that there may be a preexisting version of fhr_dir from previous -# runs of this script for the current forecast hour (and current cycle), -# e.g. from the workflow task that runs this script failing and then being -# called again. Thus, we first make sure preexisting versions are deleted. +# Note that there may be a preexisting version of tmp_dir from previous +# runs of this script for the current forecast hour (and current forecast +# start time), e.g. from the workflow task that runs this script failing +# and then being called again. Thus, we first make sure preexisting +# versions are deleted. # #----------------------------------------------------------------------- # @@ -88,11 +97,15 @@ else fi mkdir_vrfy -p "${postprd_dir}" -fhr_dir="${postprd_dir}/$fhr$fmn" -check_for_preexist_dir_file "${fhr_dir}" "delete" -mkdir_vrfy -p "${fhr_dir}" +if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then +tmp_dir="${postprd_dir}/$fhr$fmn" +else +tmp_dir="${postprd_dir}/$fhr" +fi +check_for_preexist_dir_file "${tmp_dir}" "delete" +mkdir_vrfy -p "${tmp_dir}" -cd_vrfy "${fhr_dir}" +cd_vrfy "${tmp_dir}" # #----------------------------------------------------------------------- # @@ -110,19 +123,6 @@ fi # #----------------------------------------------------------------------- # -# check if all necessary environmental variables are included -# -#----------------------------------------------------------------------- -# -if [ -n "${fmn}" ] && [ -z "${DT_ATMOS}" ]; then - print_err_msg_exit "\ -ERROR: Environmental variable \$fmn is present, so \$dt_atmos is also required -to be set (use same value as in FV3 namelist). This is needed so that -UPP will process the correct file for the first output timestep model fields." -fi -# -#----------------------------------------------------------------------- -# # Call the ex-script for this J-job and pass to it the necessary varia- # bles. # @@ -132,7 +132,7 @@ $SCRIPTSDIR/exregional_run_post.sh \ cdate="${CDATE}" \ run_dir="${run_dir}" \ postprd_dir="${postprd_dir}" \ - fhr_dir="${fhr_dir}" \ + tmp_dir="${tmp_dir}" \ fhr="${fhr}" \ fmn="${fmn}" \ dt_atmos="${DT_ATMOS}" || \ diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 83b24f3a1f..9d4ba1c090 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -59,7 +59,7 @@ valid_args=( \ "cdate" \ "run_dir" \ "postprd_dir" \ -"fhr_dir" \ +"tmp_dir" \ "fhr" \ "fmn" \ "dt_atmos" \ @@ -158,7 +158,8 @@ esac # #----------------------------------------------------------------------- # -# Remove any files from previous runs and stage necessary files in fhr_dir. +# Remove any files from previous runs and stage necessary files in the +# temporary work directory specified by tmp_dir. # #----------------------------------------------------------------------- # @@ -169,23 +170,22 @@ if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then print_info_msg " ==================================================================== Copying the user-defined post flat file specified by CUSTOM_POST_CONFIG_FP -to the post forecast hour directory (fhr_dir): +to the temporary work directory (tmp_dir): CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\" - fhr_dir = \"${fhr_dir}\" + tmp_dir = \"${tmp_dir}\" ====================================================================" else post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt" print_info_msg " ==================================================================== -Copying the default post flat file specified by post_config_fp to the post -forecast hour directory (fhr_dir): +Copying the default post flat file specified by post_config_fp to the +temporary work directory (tmp_dir): post_config_fp = \"${post_config_fp}\" - fhr_dir = \"${fhr_dir}\" + tmp_dir = \"${tmp_dir}\" ====================================================================" fi cp_vrfy ${post_config_fp} ./postxconfig-NT.txt -cp_vrfy ${EMC_POST_DIR}/parm/params_grib2_tbl_new ./params_grib2_tbl_new -cp_vrfy ${EXECDIR}/ncep_post . +cp_vrfy ${EMC_POST_DIR}/parm/params_grib2_tbl_new . # #----------------------------------------------------------------------- # @@ -212,31 +212,47 @@ tmmark="tm00" #----------------------------------------------------------------------- # # Create a text file (itag) containing arguments to pass to the post- -# processing executable. +# processor's executable. # #----------------------------------------------------------------------- # +# Set the variable (mnts_secs_str) that determines the suffix in the names +# of the forecast model's write-component output files that specifies the +# minutes and seconds of the corresponding output forecast time. +# +# Note that if the forecast model is instructed to output at some hourly +# interval (via the nfhout and nfhout_hf parameters in the MODEL_CONFIG_FN +# file, with nsout set to a non-positive value), then the write-component +# output file names will not contain any suffix for the minutes and seconds. +# For this reason, when SUB_HOURLY_POST is not set to "TRUE", mnts_sec_str +# must be set to a null string. +# mnts_secs_str="" if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then if [ ${fhr}${fmn} = "00000" ]; then - mnts_secs_str=":"`date --utc --date "${yyyymmdd} ${hh} UTC + ${dt_atmos} seconds" +%M:%S` + mnts_secs_str=":"$( date --utc --date "${yyyymmdd} ${hh} UTC + ${dt_atmos} seconds" "+%M:%S" ) else mnts_secs_str=":${fmn}:00" fi -else - fmn="00" fi - +# +# Set the names of the forecast model's write-component output files. +# dyn_file="${run_dir}/dynf${fhr}${mnts_secs_str}.nc" phy_file="${run_dir}/phyf${fhr}${mnts_secs_str}.nc" -post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours +${fmn} minutes" "+%Y%m%d%H%M" ) - +# +# Set parameters that specify the actual time (not forecast time) of the +# output. +# +post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) post_yyyy=${post_time:0:4} post_mm=${post_time:4:2} post_dd=${post_time:6:2} post_hh=${post_time:8:2} post_mn=${post_time:10:2} - +# +# Create the input text file to the post-processor executable. +# cat > itag < - {% if sub_hourly_post %} + {%- if sub_hourly_post %} 000 - 00 - + 00 + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -463,14 +463,14 @@ later below for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H fhr#fhr# - fmn#fmin# + fmn#fmn# -{# + @@ -490,7 +490,7 @@ Define the post-processing tasks for the second through last output minutes of the first output hour (which is zero). We use two metatasks for this. The inner one is needed to loop over the minutes, and the outer one is used in order to be able to declare and use the variable fhr. Use of this -variable (along with the fmin variable in the inner metatask) allows the +variable (along with the fmn variable in the inner metatask) allows the block of code inside the tag to be identical to the ones later below for other output times. --> @@ -499,16 +499,16 @@ for other output times. 000 - {% for min in range(delta_min, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} - + {% for min in range(delta_min, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -517,14 +517,14 @@ for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H fhr#fhr# - fmn#fmin# + fmn#fmn# - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc @@ -534,41 +534,45 @@ for other output times. - {% endif %} + {%- endif %} - {% if sub_hourly_post %} + {%- if sub_hourly_post %} {% for h in range(1, fcst_len_hrs) %}{{ " %03d" % h }}{% endfor %} - {% for min in range(0, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} - {% else %} + {% for min in range(0, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} + + {%- else %} {% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} - - 00 - {% endif %} - + + {%- endif %} &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + {%- if sub_hourly_post %} + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log + {%- else %} + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log + {%- endif %} GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -576,31 +580,37 @@ always zero). CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H + {%- if sub_hourly_post %} fhr#fhr# - fmn#fmin# + fmn#fmn# + {%- else %} + fhr#fhr# + {%- endif %} - {% if sub_hourly_post %} - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc - {% else %} + {%- if sub_hourly_post %} + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc + {%- else %} &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#.nc &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#.nc - {% endif %} + {%- endif %} + {%- if sub_hourly_post %} - + {%- else %} + {%- endif %} -{% if sub_hourly_post %} +{%- if sub_hourly_post %} {{ "%03d" % fcst_len_hrs }} - 00 - + 00 + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmin#_@Y@m@d@H.log + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -634,14 +644,14 @@ the tag to be identical to the ones above for other output times. SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H fhr#fhr# - fmn#fmin# + fmn#fmn# - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmin#:00.nc - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmin#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc @@ -649,9 +659,9 @@ the tag to be identical to the ones above for other output times. -{% endif %} +{%- endif %} -{% if run_task_get_obs_ccpa %} +{%- if run_task_get_obs_ccpa %} - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + &RSRV_HPSS; + {%- if machine in ["WCOSS_CRAY"] %} + + {%- endif %} + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" + {%- if machine in ["WCOSS_DELL_P3"] %} + 2048M-R affinity[core] + {%- endif %} + {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} + {{ wtime_vx_gridstat }} + &NCORES_PER_NODE; + &VX_GRIDSTAT_RETOP_TN; + &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARRETOP + + + {%- if run_task_get_obs_mrms %} + + {%- else %} + + {%- endif %} + + + +{%- endif %} + {%- if run_task_vx_gridstat %} -{% if run_task_make_grid %} +{%- if run_task_make_grid %} +{%- if run_task_run_post %} {%- if sub_hourly_post %} - &RSRV_HPSS; + &RSRV_DEFAULT; {%- if machine in ["WCOSS_CRAY"] %} {%- endif %} @@ -866,7 +866,7 @@ the tag to be identical to the ones above for other output times. --> - &RSRV_HPSS; + &RSRV_DEFAULT; {%- if machine in ["WCOSS_CRAY"] %} {%- endif %} @@ -1003,7 +1003,7 @@ the tag to be identical to the ones above for other output times. --> - &RSRV_HPSS; + &RSRV_DEFAULT; {%- if machine in ["WCOSS_CRAY"] %} {%- endif %} diff --git a/ush/templates/parm/met/GridStatConfig_APCP b/ush/templates/parm/met/GridStatConfig_APCP index 7e2f892386..4fa85ad7ee 100755 --- a/ush/templates/parm/met/GridStatConfig_APCP +++ b/ush/templates/parm/met/GridStatConfig_APCP @@ -109,6 +109,7 @@ boot = { interp = { field = BOTH; vld_thresh = 1.0; + shape = SQUARE; type = [ { @@ -151,8 +152,8 @@ output_flag = { pstd = NONE; pjc = NONE; prc = NONE; - nbrctc = NONE; - nbrcts = NONE; + nbrctc = STAT; + nbrcts = STAT; nbrcnt = STAT; } @@ -174,6 +175,6 @@ nc_pairs_flag = { rank_corr_flag = FALSE; tmp_dir = "/tmp"; output_prefix = "${OUTPUT_PREFIX}"; -version = "V9.0.2"; +version = "V10.0.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFL b/ush/templates/parm/met/GridStatConfig_REFL index 384f4bdbfd..ade839df7a 100644 --- a/ush/templates/parm/met/GridStatConfig_REFL +++ b/ush/templates/parm/met/GridStatConfig_REFL @@ -139,7 +139,7 @@ nbrhd = { field = BOTH; shape = ${NEIGHBORHOOD_SHAPE}; width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >0. ]; + cov_thresh = [ >=0.5 ]; vld_thresh = 1.0; } @@ -171,12 +171,12 @@ gradient = { // Statistical output types // output_flag = { - fho = NONE; + fho = STAT; ctc = STAT; cts = STAT; mctc = NONE; mcts = NONE; - cnt = NONE; + cnt = STAT; sl1l2 = NONE; sal1l2 = NONE; vl1l2 = NONE; @@ -214,6 +214,6 @@ grid_weight_flag = NONE; rank_corr_flag = FALSE; tmp_dir = "/tmp"; output_prefix = "${OUTPUT_PREFIX}"; -version = "V9.0.2"; +version = "V10.0.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PB2NCConfig_conus_sfc b/ush/templates/parm/met/PB2NCConfig similarity index 85% rename from ush/templates/parm/met/PB2NCConfig_conus_sfc rename to ush/templates/parm/met/PB2NCConfig index 55ce1da731..ded8359c4f 100644 --- a/ush/templates/parm/met/PB2NCConfig_conus_sfc +++ b/ush/templates/parm/met/PB2NCConfig @@ -9,7 +9,7 @@ // // PrepBufr message type // -message_type = ${PB2NC_MESSAGE_TYPE} ; +message_type = ${PB2NC_MESSAGE_TYPE}; // // Mapping of message type group name to comma-separated list of values @@ -30,7 +30,7 @@ message_type_map = []; // // PrepBufr station ID // -station_id = ${PB2NC_STATION_ID} ; +station_id = ${PB2NC_STATION_ID}; //////////////////////////////////////////////////////////////////////////////// @@ -38,8 +38,8 @@ station_id = ${PB2NC_STATION_ID} ; // Observation time window // obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; } //////////////////////////////////////////////////////////////////////////////// @@ -67,7 +67,7 @@ elevation_range = { // // Observation types // -pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; +pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; in_report_type = []; @@ -100,15 +100,8 @@ obs_bufr_var = ${OBS_BUFR_VAR_LIST}; // Mapping of input BUFR variable names to output variables names. // The default PREPBUFR map, obs_prepbufr_map, is appended to this map. // -obs_bufr_map = []; - -// -// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB -// abbreviations in the output. This default map is appended to obs_bufr_map. -// This should not typically be overridden. -// -obs_prefbufr_map = [ - { key = "HOVI"; val = "VIS"; }, +obs_bufr_map = [ + { key = "HOVI"; val = "VIS"; }, { key = "PMO"; val = "PRMSL"; }, { key = "POB"; val = "PRES"; }, { key = "QOB"; val = "SPFH"; }, @@ -117,19 +110,30 @@ obs_prefbufr_map = [ { key = "ZOB"; val = "HGT"; }, { key = "UOB"; val = "UGRD"; }, { key = "VOB"; val = "VGRD"; }, + { key = "PWO"; val = "PWAT"; }, { key = "D_DPT"; val = "DPT"; }, { key = "D_WDIR"; val = "WDIR"; }, { key = "D_WIND"; val = "WIND"; }, + { key = "MXGS"; val = "GUST"; }, { key = "D_RH"; val = "RH"; }, { key = "D_MIXR"; val = "MIXR"; }, - { key = "D_PBL"; val = "PBL"; }, - { key = "D_CAPE"; val = "CAPE"; }, - { key = "TOCC"; val = "TCDC"; } + { key = "D_PBL"; val = "PBL"; }, + { key = "D_CAPE"; val = "CAPE"; }, + { key = "CEILING"; val = "CEILING"; }, + { key = "D_PRMSL"; val = "PRMSL"; }, + { key = "TOCC"; val = "TCDC"; } ]; +// +// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB +// abbreviations in the output. This default map is appended to obs_bufr_map. +// This should not typically be overridden. +// +obs_prefbufr_map = []; + //////////////////////////////////////////////////////////////////////////////// -quality_mark_thresh = 3; +quality_mark_thresh = 9; event_stack_flag = TOP; //////////////////////////////////////////////////////////////////////////////// @@ -144,8 +148,8 @@ time_summary = { raw_data = FALSE; beg = ${TIME_SUMMARY_BEG}; end = ${TIME_SUMMARY_END}; - step = 300; - width = 600; + step = 3600; + width = 3600; grib_code = []; obs_var = ${TIME_SUMMARY_VAR_NAMES}; type = ${TIME_SUMMARY_TYPES}; @@ -156,6 +160,6 @@ time_summary = { //////////////////////////////////////////////////////////////////////////////// tmp_dir = "/tmp"; -version = "V9.0.2"; +version = "V10.0.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PB2NCConfig_upper_air b/ush/templates/parm/met/PB2NCConfig_upper_air deleted file mode 100644 index cf29a52088..0000000000 --- a/ush/templates/parm/met/PB2NCConfig_upper_air +++ /dev/null @@ -1,157 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// PB2NC configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// PrepBufr message type -// -message_type = ${PB2NC_MESSAGE_TYPE} ; - -// -// Mapping of message type group name to comma-separated list of values -// Derive PRMSL only for SURFACE message types -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } -]; - -// -// Mapping of input PrepBufr message types to output message types -// -message_type_map = []; - -// -// PrepBufr station ID -// -station_id = ${PB2NC_STATION_ID} ; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation time window -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation retention regions -// -mask = { - grid = "${PB2NC_GRID}"; - poly = "${PB2NC_POLY}"; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observing location elevation -// -elevation_range = { - beg = -1000; - end = 100000; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation types -// -pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; - -in_report_type = []; - -instrument_type = []; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Vertical levels to retain -// -level_range = { - beg = 1; - end = 511; -} - -level_category = [0, 1, 4, 5, 6]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// BUFR variable names to retain or derive. -// Use obs_bufr_map to rename variables in the output. -// If empty, process all available variables. -// -obs_bufr_var = ${OBS_BUFR_VAR_LIST}; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Mapping of input BUFR variable names to output variables names. -// The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -// -obs_bufr_map = []; - -// -// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB -// abbreviations in the output. This default map is appended to obs_bufr_map. -// This should not typically be overridden. -// -obs_prefbufr_map = [ - { key = "POB"; val = "PRES"; }, - { key = "QOB"; val = "SPFH"; }, - { key = "TOB"; val = "TMP"; }, - { key = "TDO"; val = "DPT"; }, - { key = "ZOB"; val = "HGT"; }, - { key = "UOB"; val = "UGRD"; }, - { key = "VOB"; val = "VGRD"; }, - { key = "D_DPT"; val = "DPT"; }, - { key = "D_WDIR"; val = "WDIR"; }, - { key = "D_WIND"; val = "WIND"; }, - { key = "D_RH"; val = "RH"; }, - { key = "D_MIXR"; val = "MIXR"; }, - { key = "D_PRMSL"; val = "PRMSL"; } -]; - -//////////////////////////////////////////////////////////////////////////////// - -quality_mark_thresh = 3; -event_stack_flag = TOP; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Time periods for the summarization -// obs_var (string array) is added and works like grib_code (int array) -// when use_var_id is enabled and variable names are saved. -// -time_summary = { - flag = ${TIME_SUMMARY_FLAG}; - raw_data = FALSE; - beg = ${TIME_SUMMARY_BEG}; - end = ${TIME_SUMMARY_END}; - step = 300; - width = 600; - grib_code = []; - obs_var = ${TIME_SUMMARY_VAR_NAMES}; - type = ${TIME_SUMMARY_TYPES}; - vld_freq = 0; - vld_thresh = 0.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -version = "V9.0.2"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_conus_sfc b/ush/templates/parm/met/PointStatConfig similarity index 97% rename from ush/templates/parm/met/PointStatConfig_conus_sfc rename to ush/templates/parm/met/PointStatConfig index ae1350cf45..453f10c2eb 100644 --- a/ush/templates/parm/met/PointStatConfig_conus_sfc +++ b/ush/templates/parm/met/PointStatConfig @@ -49,6 +49,10 @@ rank_corr_flag = FALSE; // // Forecast and observation fields to be verified // + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + fcst = { field = [ ${FCST_FIELD} ]; }; @@ -231,9 +235,9 @@ hira = { // May be set separately in each "obs.field" entry // output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; + fho = STAT; + ctc = STAT; + cts = STAT; mctc = NONE; mcts = NONE; cnt = STAT; @@ -254,6 +258,6 @@ output_flag = { tmp_dir = "/tmp"; output_prefix = "${OUTPUT_PREFIX}"; -version = "V9.0.2"; +version = "V10.0.0"; //////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_upper_air b/ush/templates/parm/met/PointStatConfig_upper_air deleted file mode 100644 index 0dff52c03c..0000000000 --- a/ush/templates/parm/met/PointStatConfig_upper_air +++ /dev/null @@ -1,262 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Point-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -rank_corr_flag = FALSE; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; - }; - -obs = { - field = [ ${OBS_FIELD} ]; - }; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = ${POINT_STAT_MESSAGE_TYPE}; -sid_exc = []; -obs_quality = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, - { key = "LANDSF"; val = "ADPSFC,MSONET"; }, - { key = "WATERSF"; val = "SFCSHP"; } -]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = NEAREST; - match_month = TRUE; - match_day = TRUE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; -write_cdf_bins = FALSE; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Land/Sea mask -// For LANDSF message types, only use forecast grid points where land = TRUE. -// For WATERSF message types, only use forecast grid points where land = FALSE. -// land_mask.flag may be set separately in each "obs.field" entry. -// -land_mask = { - flag = FALSE; - file_name = []; - field = { name = "LAND"; level = "L0"; } - regrid = { method = NEAREST; width = 1; } - thresh = eq1; -} - -// -// Topography -// For SURFACE message types, only use observations where the topo - station -// elevation difference meets the use_obs_thresh threshold. -// For the observations kept, when interpolating forecast data to the -// observation location, only use forecast grid points where the topo - station -// difference meets the interp_fcst_thresh threshold. -// topo_mask.flag may be set separately in each "obs.field" entry. -// -topo_mask = { - flag = FALSE; - file_name = []; - field = { name = "TOPO"; level = "L0"; } - regrid = { method = BILIN; width = 2; } - use_obs_thresh = ge-100&&le100; - interp_fcst_thresh = ge-50&&le50; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// May be set separately in each "obs.field" entry -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// May be set separately in each "obs.field" entry -// -mask = { - grid = ${POINT_STAT_GRID}; -// poly = [ -// "${FIXverif_global}/vx_mask_files/grid2obs/NA.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/SA.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/NH.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/SH.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/TRO.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/CONUS.nc" -// ]; - poly = ${POINT_STAT_POLY}; - sid = ${POINT_STAT_STATION_ID}; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// May be set separately in each "obs.field" entry -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// May be set separately in each "obs.field" entry -// -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = BILIN; - width = 2; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// HiRA verification method -// May be set separately in each "obs.field" entry -// -hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; - vld_thresh = 1.0; - cov_thresh = [ ==0.25 ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// May be set separately in each "obs.field" entry -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = STAT; - sal1l2 = NONE; - vl1l2 = STAT; - val1l2 = NONE; - vcnt = STAT; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - mpr = NONE; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V9.0.2"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index c1f75d3055..69f315fe31 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -59,7 +59,7 @@ PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} # Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig_conus_sfc +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig # If set to True, skip run if the output file determined by the output directory and # filename template already exists @@ -69,7 +69,7 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL @@ -88,7 +88,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # References PARM_BASE which is the location of the parm directory corresponding # to the ush directory of the master_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_conus_sfc +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]} @@ -108,7 +108,7 @@ POINT_STAT_GRID = # environment variable that is too long, resulting in an error. For long # lists of poly masking files (i.e. all the mask files in the NCEP_mask # directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, @@ -124,50 +124,37 @@ BOTH_VAR1_LEVELS = Z2 BOTH_VAR2_NAME = DPT BOTH_VAR2_LEVELS = Z2 -BOTH_VAR3_NAME = UGRD -BOTH_VAR3_LEVELS = Z10 -BOTH_VAR3_THRESH = >=2.572 ;; m/s or 5kts +BOTH_VAR3_NAME = RH +BOTH_VAR3_LEVELS = Z2 -BOTH_VAR4_NAME = VGRD +BOTH_VAR4_NAME = UGRD BOTH_VAR4_LEVELS = Z10 BOTH_VAR4_THRESH = >=2.572 ;; m/s or 5kts -BOTH_VAR5_NAME = PRMSL -BOTH_VAR5_LEVELS = Z0 +BOTH_VAR5_NAME = VGRD +BOTH_VAR5_LEVELS = Z10 +BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts -BOTH_VAR6_NAME = TCDC -BOTH_VAR6_LEVELS = L0 -BOTH_VAR6_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } +BOTH_VAR6_NAME = PRMSL +BOTH_VAR6_LEVELS = Z0 -BOTH_VAR7_NAME = VIS +BOTH_VAR7_NAME = TCDC BOTH_VAR7_LEVELS = L0 -BOTH_VAR7_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 -BOTH_VAR7_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } - -FCST_VAR8_NAME = HGT -FCST_VAR8_LEVELS = L0 -FCST_VAR8_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; -FCST_VAR8_THRESH = <152, <305, <914, <1520, <3040, >=914 -OBS_VAR8_NAME = CEILING -OBS_VAR8_LEVELS = L0 -OBS_VAR8_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } -OBS_VAR8_THRESH = <152, <305, <914, <1520, <3040, >=914 - -FCST_VAR9_NAME = CAPE -FCST_VAR9_LEVELS = L0 -FCST_VAR9_OPTIONS = cnt_thresh = [ >0 ]; -FCST_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 -OBS_VAR9_NAME = CAPE -OBS_VAR9_LEVELS = L0-100000 -OBS_VAR9_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; -OBS_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 +BOTH_VAR7_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +BOTH_VAR8_NAME = VIS +BOTH_VAR8_LEVELS = L0 +BOTH_VAR8_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 +BOTH_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +BOTH_VAR9_NAME = GUST +BOTH_VAR9_LEVELS = Z0 FCST_VAR10_NAME = HGT FCST_VAR10_LEVELS = L0 -FCST_VAR10_OPTIONS = GRIB_lvl_typ = 220; -OBS_VAR10_NAME = PBL +FCST_VAR10_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; +FCST_VAR10_THRESH = <152, <305, <914, <1520, <3040, >=914 +OBS_VAR10_NAME = CEILING OBS_VAR10_LEVELS = L0 -OBS_VAR10_OPTIONS = desc = "RI"; - -BOTH_VAR11_NAME = GUST -BOTH_VAR11_LEVELS = Z0 +OBS_VAR10_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } +OBS_VAR10_THRESH = <152, <305, <914, <1520, <3040, >=914 diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index aa58acdac0..bed01d8e6c 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -15,7 +15,7 @@ LOG_DIR = {OUTPUT_BASE}/log [filename_templates] # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc @@ -24,6 +24,7 @@ FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HH OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat + [config] # List of applications to run @@ -58,7 +59,7 @@ PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} # Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig_upper_air +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig # If set to True, skip run if the output file determined by the output directory and # filename template already exists @@ -68,11 +69,11 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, D_RH, D_CAPE, D_PBL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS # For defining the time periods for summarization # False for no time summary, True otherwise @@ -87,7 +88,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # References PARM_BASE which is the location of the parm directory corresponding # to the ush directory of the master_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_upper_air +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]} @@ -107,7 +108,7 @@ POINT_STAT_GRID = # environment variable that is too long, resulting in an error. For long # lists of poly masking files (i.e. all the mask files in the NCEP_mask # directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, @@ -121,40 +122,43 @@ BOTH_VAR1_NAME = TMP BOTH_VAR1_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 BOTH_VAR2_NAME = RH -BOTH_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300 - -BOTH_VAR2_NAME = DPT -BOTH_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300 +BOTH_VAR2_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250 -BOTH_VAR3_NAME = UGRD -BOTH_VAR3_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 -BOTH_VAR3_THRESH = >=2.572 ;; m/s or 5kts +BOTH_VAR3_NAME = DPT +BOTH_VAR3_LEVELS = P1000, P925, P850, P700, P500, P400, P300 -BOTH_VAR4_NAME = VGRD +BOTH_VAR4_NAME = UGRD BOTH_VAR4_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 BOTH_VAR4_THRESH = >=2.572 ;; m/s or 5kts -BOTH_VAR5_NAME = HGT -BOTH_VAR5_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - -FCST_VAR6_NAME = CAPE -FCST_VAR6_LEVELS = L0 -FCST_VAR6_OPTIONS = cnt_thresh = [ >0 ]; -FCST_VAR6_THRESH = >500, >1000, >1500, >2000, >3000, >4000 -OBS_VAR6_NAME = CAPE -OBS_VAR6_LEVELS = L0-100000 -OBS_VAR6_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; -OBS_VAR6_THRESH = >500, >1000, >1500, >2000, >3000, >4000 - -FCST_VAR7_NAME = HPBL -FCST_VAR7_LEVELS = Z0 -OBS_VAR7_NAME = PBL -OBS_VAR7_LEVELS = L0 -OBS_VAR7_OPTIONS = desc = "TKE"; - -FCST_VAR8_NAME = HGT +BOTH_VAR5_NAME = VGRD +BOTH_VAR5_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 +BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR6_NAME = HGT +BOTH_VAR6_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 + +BOTH_VAR7_NAME = SPFH +BOTH_VAR7_LEVELS = P1000, P850, P700, P500, P400, P300 + +FCST_VAR8_NAME = CAPE FCST_VAR8_LEVELS = L0 -FCST_VAR8_OPTIONS = GRIB_lvl_typ = 220; -OBS_VAR8_NAME = PBL -OBS_VAR8_LEVELS = L0 -OBS_VAR8_OPTIONS = desc = "RI"; +FCST_VAR8_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR8_THRESH = >500, >1000, >1500, >2000, >3000, >4000 +OBS_VAR8_NAME = CAPE +OBS_VAR8_LEVELS = L0-100000 +OBS_VAR8_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR8_THRESH = >500, >1000, >1500, >2000, >3000, >4000 + +FCST_VAR9_NAME = HPBL +FCST_VAR9_LEVELS = Z0 +OBS_VAR9_NAME = PBL +OBS_VAR9_LEVELS = L0 +OBS_VAR9_OPTIONS = desc = "TKE"; + +FCST_VAR10_NAME = HGT +FCST_VAR10_LEVELS = L0 +FCST_VAR10_OPTIONS = GRIB_lvl_typ = 220; +OBS_VAR10_NAME = PBL +OBS_VAR10_LEVELS = L0 +OBS_VAR10_OPTIONS = desc = "RI"; From 0b9d1497eac2769326ccc09754351753de0317d8 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 2 Jul 2021 13:00:24 -0400 Subject: [PATCH 052/203] Update met and workflow template for wcoss (#535) --- modulefiles/tasks/wcoss_dell_p3/run_vx.local | 2 +- ush/templates/FV3LAM_wflow.xml | 26 +------------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/modulefiles/tasks/wcoss_dell_p3/run_vx.local b/modulefiles/tasks/wcoss_dell_p3/run_vx.local index 7c2bf951c3..35809d9bf6 100644 --- a/modulefiles/tasks/wcoss_dell_p3/run_vx.local +++ b/modulefiles/tasks/wcoss_dell_p3/run_vx.local @@ -3,4 +3,4 @@ module unload netcdf module use /gpfs/dell2/emc/verification/noscrub/emc.metplus/modulefiles -module load met/9.1 +module load met/10.0.0 diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index fd67413ecb..3f89b0c309 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -784,13 +784,7 @@ the tag to be identical to the ones above for other output times. &RSRV_DEFAULT; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; @@ -826,13 +820,7 @@ the tag to be identical to the ones above for other output times. &RSRV_DEFAULT; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; @@ -861,19 +849,13 @@ the tag to be identical to the ones above for other output times. {%- if run_task_vx_gridstat %} &RSRV_DEFAULT; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; @@ -1004,13 +986,7 @@ the tag to be identical to the ones above for other output times. &RSRV_DEFAULT; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_POINTSTAT" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} {{ wtime_vx_pointstat }} &NCORES_PER_NODE; From aaecc03480c293195a702bcd9e5acc5606a81161 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 2 Jul 2021 17:34:40 -0400 Subject: [PATCH 053/203] Make the coupling parameter CPL in model_configure selectable (#523) * Set cpl in setup.sh * Remove cpl from config_default.sh --- ush/config_defaults.sh | 34 ++++++++++++++++---------- ush/create_model_configure_file.sh | 2 ++ ush/setup.sh | 38 +++++++++++++++++++++++++----- ush/templates/model_configure | 2 +- ush/valid_param_vals.sh | 1 + 5 files changed, 58 insertions(+), 19 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 05090b19c1..d22cf202d4 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -380,8 +380,28 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # FCST_LEN_HRS: # The length of each forecast, in integer hours. # +#----------------------------------------------------------------------- +# +DATE_FIRST_CYCL="YYYYMMDD" +DATE_LAST_CYCL="YYYYMMDD" +CYCL_HRS=( "HH1" "HH2" ) +FCST_LEN_HRS="24" +# +#----------------------------------------------------------------------- +# +# Set model_configure parameters. Definitions: +# +# DT_ATMOS: +# The main forecast model integraton time step. As described in the +# forecast model documentation, "It corresponds to the frequency with +# which the top level routine in the dynamics is called as well as the +# frequency with which the physics is called." +# +# CPL: parameter for coupling +# (set automatically based on FCST_MODEL in ush/setup.sh) +# (ufs-weather-model:FALSE, fv3gfs_aqm:TRUE) +# # RESTART_INTERVAL: -# Parameter in model_configure; # frequency of the output restart files (unit:hour). # Default=0: restart files are produced at the end of a forecast run # For example, RESTART_INTERVAL="1": restart files are produced every hour @@ -389,10 +409,7 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # #----------------------------------------------------------------------- # -DATE_FIRST_CYCL="YYYYMMDD" -DATE_LAST_CYCL="YYYYMMDD" -CYCL_HRS=( "HH1" "HH2" ) -FCST_LEN_HRS="24" +DT_ATMOS="" RESTART_INTERVAL="0" # #----------------------------------------------------------------------- @@ -935,12 +952,6 @@ ESGgrid_WIDE_HALO_WIDTH="" # # Set computational parameters for the forecast. Definitions: # -# DT_ATMOS: -# The main forecast model integraton time step. As described in the -# forecast model documentation, "It corresponds to the frequency with -# which the top level routine in the dynamics is called as well as the -# frequency with which the physics is called." -# # LAYOUT_X, LAYOUT_Y: # The number of MPI tasks (processes) to use in the two horizontal # directions (x and y) of the regional grid when running the forecast @@ -969,7 +980,6 @@ ESGgrid_WIDE_HALO_WIDTH="" # #----------------------------------------------------------------------- # -DT_ATMOS="" LAYOUT_X="" LAYOUT_Y="" BLOCKSIZE="" diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index bb9760c02b..94902ba96b 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -102,6 +102,7 @@ run directory (run_dir): # dot_quilting_dot="."${QUILTING,,}"." dot_print_esmf_dot="."${PRINT_ESMF,,}"." + dot_cpl_dot="."${CPL,,}"." # #----------------------------------------------------------------------- # @@ -119,6 +120,7 @@ run directory (run_dir): 'start_hour': $hh 'nhours_fcst': ${FCST_LEN_HRS} 'dt_atmos': ${DT_ATMOS} + 'cpl': ${dot_cpl_dot} 'atmos_nthreads': ${nthreads:-1} 'ncores_per_node': ${NCORES_PER_NODE} 'restart_interval': ${RESTART_INTERVAL} diff --git a/ush/setup.sh b/ush/setup.sh index b7370ce5cb..cfd31ffeb4 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -636,6 +636,30 @@ check_var_valid_value \ # #----------------------------------------------------------------------- # +# Make sure that FCST_MODEL is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "FCST_MODEL" "valid_vals_FCST_MODEL" +# +#----------------------------------------------------------------------- +# +# Set CPL to TRUE/FALSE based on FCST_MODEL. +# +#----------------------------------------------------------------------- +# +if [ "${FCST_MODEL}" = "ufs-weather-model" ]; then + CPL="FALSE" +elif [ "${FCST_MODEL}" = "fv3gfs_aqm" ]; then + CPL="TRUE" +else + print_err_msg_exit "\ +The coupling flag CPL has not been specified for this value of FCST_MODEL: + FCST_MODEL = \"${FCST_MODEL}\"" +fi +# +#----------------------------------------------------------------------- +# # Make sure RESTART_INTERVAL is set to an integer value if present # #----------------------------------------------------------------------- @@ -2366,12 +2390,6 @@ str_to_insert=${str_to_insert//$'\n'/\\n} # regexp="(^#!.*)" sed -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} - - - - - - # # Loop through the lines in line_list. # @@ -2735,6 +2753,14 @@ fi # #----------------------------------------------------------------------- # +# CPL: parameter for coupling in model_configure +# +#----------------------------------------------------------------------- +# +CPL="${CPL}" +# +#----------------------------------------------------------------------- +# # Name of the ozone parameterization. The value this gets set to depends # on the CCPP physics suite being used. # diff --git a/ush/templates/model_configure b/ush/templates/model_configure index c9be0cf7d8..f97edc90fa 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -10,7 +10,7 @@ nhours_fcst: {{ nhours_fcst }} RUN_CONTINUE: .false. ENS_SPS: .false. dt_atmos: {{ dt_atmos }} -cpl: .false. +cpl: {{ cpl }} calendar: 'julian' memuse_verbose: .false. atmos_nthreads: {{ atmos_nthreads }} diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 64ec0546ff..4cc667c3c3 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -2,6 +2,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") +valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_PREDEF_GRID_NAME=( \ "RRFS_CONUS_25km" \ "RRFS_CONUS_13km" \ From 4a2a85dee5f3bc1112d8a509b3409d17fbb74947 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 7 Jul 2021 11:51:27 -0400 Subject: [PATCH 054/203] Add inline post option to the SRW App (#538) * Add write_dopost for inline post * Add inline_post to run_fcst script * Turn off run_post when true * Add postprod to run_fcst script * Update workflow template for inline post * Add a WE2E test for inline post --- scripts/exregional_run_fcst.sh | 65 ++++++++++++++++++-- tests/baseline_configs/config.inline_post.sh | 28 +++++++++ tests/baselines_list.txt | 1 + ush/config_defaults.sh | 5 ++ ush/create_model_configure_file.sh | 2 + ush/generate_FV3LAM_wflow.sh | 4 ++ ush/setup.sh | 24 ++++++++ ush/templates/FV3LAM_wflow.xml | 28 +++++++++ ush/templates/model_configure | 1 + ush/valid_param_vals.sh | 1 + 10 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 tests/baseline_configs/config.inline_post.sh diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 4268a8899f..f1e164eebd 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -250,8 +250,6 @@ Cannot create symlink because target does not exist: target = \"$target\"" fi - - relative_or_null="" if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then relative_or_null="--relative" @@ -290,7 +288,6 @@ Cannot create symlink because target does not exist: target = \"$target\"" fi - # # If using the FV3_HRRR physics suite, there are two files (that contain # statistics of the orography) that are needed by the gravity wave drag @@ -317,7 +314,6 @@ Cannot create symlink because target does not exist: fi - # #----------------------------------------------------------------------- # @@ -444,6 +440,27 @@ ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${run_dir} ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${run_dir} ln_vrfy -sf ${relative_or_null} ${NEMS_CONFIG_FP} ${run_dir} + +if [ ${WRITE_DOPOST} = "TRUE" ]; then + cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat + if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then + post_config_fp="${CUSTOM_POST_CONFIG_FP}" + print_info_msg " +==================================================================== + CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\" +====================================================================" + else + post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt" + print_info_msg " +==================================================================== + post_config_fp = \"${post_config_fp}\" +====================================================================" + fi + cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt + cp_vrfy ${post_config_fp} ./postxconfig-NT.txt + cp_vrfy ${EMC_POST_DIR}/parm/params_grib2_tbl_new . +fi + if [ "${DO_ENSEMBLE}" = TRUE ]; then set_FV3nml_stoch_params cdate="$cdate" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current @@ -507,6 +524,46 @@ $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # +# Only for inline post, create the directory where post-processing output +# are stored (postprd_dir) +if [ ${WRITE_DOPOST} = "TRUE" ]; then + if [ "${RUN_ENVIR}" = "nco" ]; then + COMOUT="${COMOUT_BASEDIR}/$RUN.$PDY/$cyc${SLASH_ENSMEM_SUBDIR}" + postprd_dir="$COMOUT" + else + postprd_dir="${run_dir}/postprd" + fi + mkdir_vrfy -p "${postprd_dir}" + + cd_vrfy ${postprd_dir} + + yyyymmdd=${cdate:0:8} + hh=${cdate:8:2} + cyc=$hh + tmmark="tm00" + fmn="00" + + for fhr in $(seq -f "%02g" 0 ${FCST_LEN_HRS}); do + echo $fhr + post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) + post_mn=${post_time:10:2} + post_mn_or_null="" + post_fn_suffix="GrbF${fhr}" + post_renamed_fn_suffix="f0${fhr}${post_mn_or_null}.${tmmark}.grib2" + + basetime=$( date --date "$yyyymmdd $hh" +%y%j%H%M ) + symlink_suffix="_${basetime}f0${fhr}${post_mn}" + fids=( "bgdawp" "bgrd3d" ) + for fid in "${fids[@]}"; do + FID="${fid^^}" + post_orig_fn="${FID}.${post_fn_suffix}" + post_renamed_fn="${NET}.t${cyc}z.${fid}${post_renamed_fn_suffix}" + mv_vrfy ../${post_orig_fn} ${post_renamed_fn} + ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} + done + done +fi +# #----------------------------------------------------------------------- # # Print message indicating successful completion of script. diff --git a/tests/baseline_configs/config.inline_post.sh b/tests/baseline_configs/config.inline_post.sh new file mode 100644 index 0000000000..1e04453069 --- /dev/null +++ b/tests/baseline_configs/config.inline_post.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to use the inline +# post option (WRITE_DOPOST) in model_configure. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +WRITE_DOPOST="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + + diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 90cd4acd90..9a9b73d898 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -53,6 +53,7 @@ grid_RRFS_NA_3km grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +inline_post nco_ensemble nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index d22cf202d4..1299a86709 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -407,10 +407,15 @@ FCST_LEN_HRS="24" # For example, RESTART_INTERVAL="1": restart files are produced every hour # with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory # +# WRITE_DOPOST: +# Flag that determines whether or not to use the INLINE POST option +# When TRUE, force to turn off run_post (RUN_TASK_RUN_POST=FALSE) in setup.sh +# #----------------------------------------------------------------------- # DT_ATMOS="" RESTART_INTERVAL="0" +WRITE_DOPOST="FALSE" # #----------------------------------------------------------------------- # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index 94902ba96b..1be54516ea 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -103,6 +103,7 @@ run directory (run_dir): dot_quilting_dot="."${QUILTING,,}"." dot_print_esmf_dot="."${PRINT_ESMF,,}"." dot_cpl_dot="."${CPL,,}"." + dot_write_dopost="."${WRITE_DOPOST,,}"." # #----------------------------------------------------------------------- # @@ -124,6 +125,7 @@ run directory (run_dir): 'atmos_nthreads': ${nthreads:-1} 'ncores_per_node': ${NCORES_PER_NODE} 'restart_interval': ${RESTART_INTERVAL} + 'write_dopost': ${dot_write_dopost} 'quilting': ${dot_quilting_dot} 'print_esmf': ${dot_print_esmf_dot} 'output_grid': ${WRTCMP_output_grid}" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index cf2575361b..2def5f81b6 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -321,6 +321,10 @@ settings="\ # 'fcst_len_hrs': ${FCST_LEN_HRS} # +# Inline post +# + 'write_dopost': ${WRITE_DOPOST} +# # METPlus-specific information # 'model': ${MODEL} diff --git a/ush/setup.sh b/ush/setup.sh index cfd31ffeb4..49202ff782 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2121,6 +2121,30 @@ fi # #----------------------------------------------------------------------- # +# Make sure that WRITE_DOPOST is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "WRITE_DOPOST" "valid_vals_WRITE_DOPOST" +# +# Set WRITE_DOPOST to either "TRUE" or "FALSE" so we don't have to consider +# other valid values later on. +# +WRITE_DOPOST=${WRITE_DOPOST^^} +if [ "$WRITE_DOPOST" = "TRUE" ] || \ + [ "$WRITE_DOPOST" = "YES" ]; then + WRITE_DOPOST="TRUE" + +# Turn off run_post + RUN_TASK_RUN_POST="FALSE" + +elif [ "$WRITE_DOPOST" = "FALSE" ] || \ + [ "$WRITE_DOPOST" = "NO" ]; then + WRITE_DOPOST="FALSE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that QUILTING is set to a valid value. # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 3f89b0c309..5198552665 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -693,7 +693,11 @@ the tag to be identical to the ones above for other output times. ACCUM01 + {%- if write_dopost %} + + {%- else %} + {%- endif %} @@ -732,7 +736,11 @@ the tag to be identical to the ones above for other output times. VARREFC RETOP + {%- if write_dopost %} + + {%- else %} + {%- endif %} @@ -769,7 +777,11 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + {%- if write_dopost %} + + {%- else %} + {%- endif %} @@ -805,7 +817,11 @@ the tag to be identical to the ones above for other output times. {%- if run_task_get_obs_ccpa %} {%- else %} + {%- if write_dopost %} + + {%- else %} + {%- endif %} {%- endif %} @@ -840,7 +856,11 @@ the tag to be identical to the ones above for other output times. {%- if run_task_get_obs_mrms %} {%- else %} + {%- if write_dopost %} + + {%- else %} + {%- endif %} {%- endif %} @@ -875,7 +895,11 @@ the tag to be identical to the ones above for other output times. {%- if run_task_get_obs_mrms %} {%- else %} + {%- if write_dopost %} + + {%- else %} + {%- endif %} {%- endif %} @@ -1005,7 +1029,11 @@ the tag to be identical to the ones above for other output times. {%- if run_task_get_obs_ndas %} {%- else %} + {%- if write_dopost %} + + {%- else %} + {%- endif %} {%- endif %} diff --git a/ush/templates/model_configure b/ush/templates/model_configure index f97edc90fa..bdd812b750 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -20,6 +20,7 @@ debug_affinity: .true. restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. print_esmf: {{ print_esmf }} +write_dopost: {{ write_dopost }} quilting: {{ quilting }} {% if quilting %} diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 4cc667c3c3..2edcf6b56d 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -51,6 +51,7 @@ valid_vals_RUN_TASK_MAKE_SFC_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "N valid_vals_RUN_TASK_RUN_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_VX_GRIDSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_VX_POINTSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_WRITE_DOPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From 0cc0f1212b09dc0c95a82881c9c3b9f9004cdfc9 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Fri, 9 Jul 2021 14:35:48 -0600 Subject: [PATCH 055/203] Changes required to produce the diag_table file at forecast run time. (#533) * Changes required to produce the diag_table file at forecast run time. * Minor syntax changes. * Syntax bug fix and update to header. * Update create_diag_table_file.sh * Update setup.sh * Update setup.sh --- scripts/exregional_make_grid.sh | 5 ---- scripts/exregional_run_fcst.sh | 14 ++++++++++ ...ble_files.sh => create_diag_table_file.sh} | 28 ++++++++----------- ush/generate_FV3LAM_wflow.sh | 20 ------------- ush/setup.sh | 7 +++++ 5 files changed, 33 insertions(+), 41 deletions(-) rename ush/{create_diag_table_files.sh => create_diag_table_file.sh} (85%) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index fdc0088aa3..fa5fed2e78 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -19,7 +19,6 @@ . $USHDIR/make_grid_mosaic_file.sh . $USHDIR/link_fix.sh . $USHDIR/set_FV3nml_sfc_climo_filenames.sh -. $USHDIR/create_diag_table_files.sh # #----------------------------------------------------------------------- # @@ -647,10 +646,6 @@ failed." set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist file failed." - -create_diag_table_files || print_err_msg_exit "\ -Call to function to create a diagnostics table file under each cycle -directory failed." # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index f1e164eebd..44e2b9cc54 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -18,6 +18,7 @@ #----------------------------------------------------------------------- # . $USHDIR/create_model_configure_file.sh +. $USHDIR/create_diag_table_file.sh # #----------------------------------------------------------------------- # @@ -492,6 +493,19 @@ cycle's (cdate) run directory (run_dir) failed: # #----------------------------------------------------------------------- # +# Call the function that creates the diag_table file within each cycle +# directory. +# +#----------------------------------------------------------------------- +# +create_diag_table_file \ + run_dir="${run_dir}" || print_err_msg_exit "\ +Call to function to create a diag table file for the current cycle's +(cdate) run directory (run_dir) failed: + run_dir = \"${run_dir}\"" +# +#----------------------------------------------------------------------- +# # If running ensemble forecasts, create a link to the cycle-specific # diagnostic tables file in the cycle directory. Note that this link # should not be made if not running ensemble forecasts because in that diff --git a/ush/create_diag_table_files.sh b/ush/create_diag_table_file.sh similarity index 85% rename from ush/create_diag_table_files.sh rename to ush/create_diag_table_file.sh index cff8c0424e..56e22b5a67 100644 --- a/ush/create_diag_table_files.sh +++ b/ush/create_diag_table_file.sh @@ -6,7 +6,7 @@ # #----------------------------------------------------------------------- # -function create_diag_table_files() { +function create_diag_table_file() { # #----------------------------------------------------------------------- # @@ -45,7 +45,9 @@ function create_diag_table_files() { # #----------------------------------------------------------------------- # - local valid_args=() + local valid_args=( \ + "run_dir" \ + ) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -65,28 +67,24 @@ function create_diag_table_files() { #----------------------------------------------------------------------- # local i \ - cdate \ - cycle_dir \ diag_table_fp \ settings # #----------------------------------------------------------------------- # -# Create a diagnostics table file within each cycle directory. +# Create a diagnostics table file within the specified run directory. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating a diagnostics table file (\"${DIAG_TABLE_FN}\") within each cycle -directory..." +Creating a diagnostics table file (\"${DIAG_TABLE_FN}\") in the specified +run directory... - for (( i=0; i<${NUM_CYCLES}; i++ )); do + run_dir = \"${run_dir}\"" - cdate="${ALL_CDATES[$i]}" - cycle_dir="${CYCLE_BASEDIR}/$cdate" - - diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" + diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" print_info_msg "$VERBOSE" " + Using the template diagnostics table file: diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP} @@ -96,9 +94,8 @@ to create: diag_table_fp = \"${diag_table_fp}\"" settings=" - starttime: !datetime ${ALL_CDATES[$i]} - cres: ${CRES} -" +starttime: !datetime ${CDATE} +cres: ${CRES}" $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ print_err_msg_exit " @@ -109,7 +106,6 @@ fill_jinja_template.py failed! !!!!!!!!!!!!!!!!! " - done # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 2def5f81b6..1269db0ab6 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -47,7 +47,6 @@ ushdir="${scrfunc_dir}" # . $ushdir/source_util_funcs.sh . $ushdir/set_FV3nml_sfc_climo_filenames.sh -. $ushdir/create_diag_table_files.sh # #----------------------------------------------------------------------- # @@ -385,21 +384,6 @@ $settings" # #----------------------------------------------------------------------- # -# Create the cycle directories. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Creating the cycle directories..." - -for (( i=0; i<${NUM_CYCLES}; i++ )); do - cdate="${ALL_CDATES[$i]}" - cycle_dir="${CYCLE_BASEDIR}/$cdate" - mkdir_vrfy -p "${cycle_dir}" -done -# -#----------------------------------------------------------------------- -# # Create a symlink in the experiment directory that points to the workflow # (re)launch script. # @@ -796,10 +780,6 @@ if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then Call to function to set surface climatology file names in the FV3 namelist file failed." - create_diag_table_files || print_err_msg_exit "\ -Call to function to create a diagnostics table file under each cycle -directory failed." - fi # #----------------------------------------------------------------------- diff --git a/ush/setup.sh b/ush/setup.sh index 49202ff782..3fb6417ce8 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -754,6 +754,13 @@ set_cycle_dates \ output_varname_all_cdates="ALL_CDATES" NUM_CYCLES="${#ALL_CDATES[@]}" + +if [ $NUM_CYCLES -gt 30 ] ; then + unset ALL_CDATES + print_info_msg "$VERBOSE" " +Too many cycles in ALL_CDATES to list, redefining in abbreviated form." +ALL_CDATES="${DATE_FIRST_CYCL}${CYCL_HRS[0]}...${DATE_LAST_CYCL}${CYCL_HRS[-1]}" +fi # #----------------------------------------------------------------------- # From a79c9e96f04ea691aaf09de6b7158451024714ec Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 9 Jul 2021 16:44:18 -0400 Subject: [PATCH 056/203] Fix inline post issue in nco mode (#541) * Remove echo from script * Add path to result files * Add a new WE2E test for inline post in nco mode --- scripts/exregional_run_fcst.sh | 16 +++++------ .../config.nco_inline_post.sh | 28 +++++++++++++++++++ tests/baselines_list.txt | 1 + ush/setup.sh | 7 ++++- 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 tests/baseline_configs/config.nco_inline_post.sh diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 44e2b9cc54..b0ff803be4 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -541,6 +541,13 @@ code." # Only for inline post, create the directory where post-processing output # are stored (postprd_dir) if [ ${WRITE_DOPOST} = "TRUE" ]; then + + yyyymmdd=${cdate:0:8} + hh=${cdate:8:2} + cyc=$hh + tmmark="tm00" + fmn="00" + if [ "${RUN_ENVIR}" = "nco" ]; then COMOUT="${COMOUT_BASEDIR}/$RUN.$PDY/$cyc${SLASH_ENSMEM_SUBDIR}" postprd_dir="$COMOUT" @@ -551,14 +558,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cd_vrfy ${postprd_dir} - yyyymmdd=${cdate:0:8} - hh=${cdate:8:2} - cyc=$hh - tmmark="tm00" - fmn="00" - for fhr in $(seq -f "%02g" 0 ${FCST_LEN_HRS}); do - echo $fhr post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) post_mn=${post_time:10:2} post_mn_or_null="" @@ -572,7 +572,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then FID="${fid^^}" post_orig_fn="${FID}.${post_fn_suffix}" post_renamed_fn="${NET}.t${cyc}z.${fid}${post_renamed_fn_suffix}" - mv_vrfy ../${post_orig_fn} ${post_renamed_fn} + mv_vrfy ${run_dir}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done done diff --git a/tests/baseline_configs/config.nco_inline_post.sh b/tests/baseline_configs/config.nco_inline_post.sh new file mode 100644 index 0000000000..003b082340 --- /dev/null +++ b/tests/baseline_configs/config.nco_inline_post.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to use the inline +# post option (WRITE_DOPOST) in model_configure for 'nco' mode. +# + +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +WRITE_DOPOST="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + + diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 9a9b73d898..f920d9a667 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -63,6 +63,7 @@ nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR +nco_inline_post new_ESGgrid new_GFDLgrid new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE diff --git a/ush/setup.sh b/ush/setup.sh index 3fb6417ce8..1633e91efc 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2145,6 +2145,12 @@ if [ "$WRITE_DOPOST" = "TRUE" ] || \ # Turn off run_post RUN_TASK_RUN_POST="FALSE" +# Check if SUB_HOURLY_POST is on + if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then + print_err_msg_exit "\ +SUB_HOURLY_POST is NOT available with Inline Post yet." + fi + elif [ "$WRITE_DOPOST" = "FALSE" ] || \ [ "$WRITE_DOPOST" = "NO" ]; then WRITE_DOPOST="FALSE" @@ -2247,7 +2253,6 @@ fi # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - # #----------------------------------------------------------------------- # From b99f264af3107245de774e3185a8fc66b38c5cdf Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Jul 2021 14:57:02 -0600 Subject: [PATCH 057/203] Edit comments, remove unneeded variables, fix indentation. --- scripts/exregional_run_fcst.sh | 10 ++++++++-- ush/create_diag_table_file.sh | 12 +++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index b0ff803be4..97b1778adb 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -538,8 +538,13 @@ $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # -# Only for inline post, create the directory where post-processing output -# are stored (postprd_dir) +#----------------------------------------------------------------------- +# +# If doing inline post, create the directory in which post-processing +# output is stored (postprd_dir). +# +#----------------------------------------------------------------------- +# if [ ${WRITE_DOPOST} = "TRUE" ]; then yyyymmdd=${cdate:0:8} @@ -576,6 +581,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done done + fi # #----------------------------------------------------------------------- diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 56e22b5a67..10ad396ce9 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -66,8 +66,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - local i \ - diag_table_fp \ + local diag_table_fp \ settings # #----------------------------------------------------------------------- @@ -82,8 +81,8 @@ run directory... run_dir = \"${run_dir}\"" - diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" - print_info_msg "$VERBOSE" " + diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" + print_info_msg "$VERBOSE" " Using the template diagnostics table file: @@ -93,11 +92,11 @@ to create: diag_table_fp = \"${diag_table_fp}\"" - settings=" + settings=" starttime: !datetime ${CDATE} cres: ${CRES}" - $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ + $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ print_err_msg_exit " !!!!!!!!!!!!!!!!! @@ -105,7 +104,6 @@ fill_jinja_template.py failed! !!!!!!!!!!!!!!!!! " - # #----------------------------------------------------------------------- # From b56423937eed14ab0ede54f9d0325b19362159c8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 12 Jul 2021 16:44:20 +0000 Subject: [PATCH 058/203] Revert "Edit comments, remove unneeded variables, fix indentation." This reverts commit b99f264af3107245de774e3185a8fc66b38c5cdf. Commit was made to develop branch of authoritative repository in error; settings have now been changed to disallow direct commits to develop and release branches to avoid this problem in the future. --- scripts/exregional_run_fcst.sh | 10 ++-------- ush/create_diag_table_file.sh | 12 +++++++----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 97b1778adb..b0ff803be4 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -538,13 +538,8 @@ $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # -#----------------------------------------------------------------------- -# -# If doing inline post, create the directory in which post-processing -# output is stored (postprd_dir). -# -#----------------------------------------------------------------------- -# +# Only for inline post, create the directory where post-processing output +# are stored (postprd_dir) if [ ${WRITE_DOPOST} = "TRUE" ]; then yyyymmdd=${cdate:0:8} @@ -581,7 +576,6 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done done - fi # #----------------------------------------------------------------------- diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 10ad396ce9..56e22b5a67 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -66,7 +66,8 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - local diag_table_fp \ + local i \ + diag_table_fp \ settings # #----------------------------------------------------------------------- @@ -81,8 +82,8 @@ run directory... run_dir = \"${run_dir}\"" - diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" - print_info_msg "$VERBOSE" " + diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" + print_info_msg "$VERBOSE" " Using the template diagnostics table file: @@ -92,11 +93,11 @@ to create: diag_table_fp = \"${diag_table_fp}\"" - settings=" + settings=" starttime: !datetime ${CDATE} cres: ${CRES}" - $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ + $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ print_err_msg_exit " !!!!!!!!!!!!!!!!! @@ -104,6 +105,7 @@ fill_jinja_template.py failed! !!!!!!!!!!!!!!!!! " + # #----------------------------------------------------------------------- # From 56d025668624f5083713b9c8e90560f6b092c2f4 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Tue, 13 Jul 2021 00:11:13 -0600 Subject: [PATCH 059/203] Streamline symlink creation (#536) ## DESCRIPTION OF CHANGES: This PR (1) helps reduce code duplication and (2) removes the necessity to check for the machine the workflow is running on before attempting to create relative symlinks by introducing the new function `create_symlink_to_file` for creating symlinks to files (but not directories). This function: * Checks for the existence of a symlink's target before creating it. If the target doesn't exist, it prints out an error message and exits. If it does, it creates the symlink to the target file. Currently, checking for the existence of a target is done in several places in the scripts. This PR changes the code so that instead of performing this check themselves, the scripts call this new function to perform this test and, if the test succeeds, create the symlink. * Creates a relative symlink (i.e. a symlink whose target is specified via a relative path) only if requested via an input argument AND if the machine/OS/shell the workflow is running on supports relative symlinks. The second half of this test (after the AND) is done implicitly via the use of the new experiment variable `RELATIVE_LINK_FLAG`. This variable gets set during experiment generation to the flag that needs to be passed to the link command on the machine in order to create relative links. For example, on Hera, `RELATIVE_LINK_FLAG` gets set to "--relative". On machines that don't support creation of relative links, `RELATIVE_LINK_FLAG` gets set to a null string. ## TESTS CONDUCTED: The following WE2E tests were conducted: ``` DOT_OR_USCORE nco_ensemble nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR pregen_grid_orog_sfc_climo suite_FV3_GFS_v15p2 suite_FV3_GFS_v16 ``` All completed successfully except suite_FV3_GFS_v16. The latter failed in the run_fcst task after hour 2, i.e. all the symlinks, etc were created properly for the forecast to start, so this failure was not due to changes in this PR and was preexisting (confirmed this by running the test with the develop branch). --- scripts/exregional_make_grid.sh | 2 +- scripts/exregional_make_orog.sh | 10 +- scripts/exregional_make_sfc_climo.sh | 2 +- scripts/exregional_run_fcst.sh | 201 ++++++++++++----------- scripts/exregional_run_post.sh | 10 +- ush/bash_utils/create_symlink_to_file.sh | 163 ++++++++++++++++++ ush/bash_utils/filesys_cmds_vrfy.sh | 52 +++--- ush/constants.sh | 14 +- ush/create_diag_table_file.sh | 12 +- ush/generate_FV3LAM_wflow.sh | 6 +- ush/link_fix.sh | 101 +++++------- ush/set_FV3nml_stoch_params.sh | 28 ++-- ush/setup.sh | 46 +++++- ush/source_util_funcs.sh | 23 ++- ush/valid_param_vals.sh | 5 +- 15 files changed, 444 insertions(+), 231 deletions(-) create mode 100644 ush/bash_utils/create_symlink_to_file.sh diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index fa5fed2e78..b68259310e 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -83,7 +83,7 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") { save_shell_opts; set +x; } > /dev/null 2>&1 diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index a512bb340d..dfd77bc7c3 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -95,7 +95,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") { save_shell_opts; set +x; } > /dev/null 2>&1 @@ -479,12 +479,8 @@ cp_vrfy "${raw_orog_fp}" "${filtered_orog_fp}" # filtering executable will run) with the same name as the grid file and # point it to the actual grid file specified by grid_fp. # - -if [ "${MACHINE}" = "WCOSS_CRAY" ]; then - ln_vrfy -fs "${grid_fp}" "${filter_dir}/${grid_fn}" -else - ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}" -fi +create_symlink_to_file target="${grid_fp}" symlink="${filter_dir}/${grid_fn}" \ + relative="TRUE" # # Create the namelist file (in the filter_dir directory) that the orography diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 46fed72eeb..42f5b39416 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -136,7 +136,7 @@ EOF # #----------------------------------------------------------------------- # -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index b0ff803be4..89caf2e51a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -99,7 +99,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} # #----------------------------------------------------------------------- # -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") ulimit -s unlimited @@ -188,22 +188,29 @@ the grid and (filtered) orography files ..." cd_vrfy ${run_dir}/INPUT -relative_or_null="" -if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ]; then - relative_or_null="--relative" +# +# For experiments in which the MAKE_GRID_TN task is run, we make the +# symlinks to the grid files relative because those files wlll be located +# within the experiment directory. This keeps the experiment directory +# more portable and the symlinks more readable. However, for experiments +# in which the MAKE_GRID_TN task is not run, pregenerated grid files will +# be used, and those will be located in an arbitrary directory (specified +# by the user) that is somwehere outside the experiment directory. Thus, +# in this case, there isn't really an advantage to using relative symlinks, +# so we use symlinks with absolute paths. +# +if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ]; then + relative_link_flag="TRUE" +else + relative_link_flag="FALSE" fi # Symlink to mosaic file with a completely different name. #target="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" # Should this point to this halo4 file or a halo3 file??? target="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" # Should this point to this halo4 file or a halo3 file??? symlink="grid_spec.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" ## Symlink to halo-3 grid file with "halo3" stripped from name. #target="${FIXLAM}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" @@ -221,13 +228,8 @@ grid_fn=$( get_charvar_from_netcdf "${mosaic_fn}" "gridfiles" ) target="${FIXLAM}/${grid_fn}" symlink="${grid_fn}" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" # Symlink to halo-4 grid file with "${CRES}_" stripped from name. # @@ -243,29 +245,26 @@ fi # target="${FIXLAM}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" symlink="grid.tile${TILE_RGNL}.halo${NH4}.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" -relative_or_null="" -if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then - relative_or_null="--relative" + +# +# As with the symlinks grid files above, when creating the symlinks to +# the orography files, use relative paths if running the MAKE_OROG_TN +# task and absolute paths otherwise. +# +if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ]; then + relative_link_flag="TRUE" +else + relative_link_flag="FALSE" fi # Symlink to halo-0 orography file with "${CRES}_" and "halo0" stripped from name. target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" # # Symlink to halo-4 orography file with "${CRES}_" stripped from name. # @@ -281,14 +280,8 @@ fi # target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" symlink="oro_data.tile${TILE_RGNL}.halo${NH4}.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi - +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" # # If using the FV3_HRRR physics suite, there are two files (that contain # statistics of the orography) that are needed by the gravity wave drag @@ -303,14 +296,8 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then for fileid in "${fileids[@]}"; do target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data_${fileid}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data_${fileid}.nc" - if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink - else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"${target}\" - symlink = \"${symlink}\"" - fi + create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" done fi @@ -339,30 +326,21 @@ of the current run directory (run_dir), where ..." cd_vrfy ${run_dir}/INPUT -#ln_vrfy -sf gfs_data.tile${TILE_RGNL}.halo${NH0}.nc gfs_data.nc -#ln_vrfy -sf sfc_data.tile${TILE_RGNL}.halo${NH0}.nc sfc_data.nc - -relative_or_null="" +# +# The symlinks to be created point to files in the same directory (INPUT), +# so it's most straightforward to use relative paths. +# +relative_link_flag="TRUE" target="gfs_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="gfs_data.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" target="sfc_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="sfc_data.nc" -if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink -else - print_err_msg_exit "\ -Cannot create symlink because target does not exist: - target = \"$target\"" -fi +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" # #----------------------------------------------------------------------- # @@ -381,10 +359,22 @@ Creating links in the current run directory (run_dir) to fixed (i.e. static) files in the FIXam directory: FIXam = \"${FIXam}\" run_dir = \"${run_dir}\"" - -relative_or_null="" -if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then - relative_or_null="--relative" +# +# For experiments that are run in "community" mode, the FIXam directory +# is an actual directory (i.e. not a symlink) located under the experiment +# directory containing actual files (i.e. not symlinks). In this case, +# we use relative paths for the symlinks in order to keep the experiment +# directory more portable and the symlinks more readable. However, for +# experiments that are run in "nco" mode, the FIXam directory is a symlink +# under the experiment directory that points to an arbitrary (user specified) +# location outside the experiment directory. Thus, in this case, there +# isn't really an advantage to using relative symlinks, so we use symlinks +# with absolute paths. +# +if [ "${RUN_ENVIR}" != "nco" ]; then + relative_link_flag="TRUE" +else + relative_link_flag="FALSE" fi regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" @@ -399,13 +389,8 @@ for (( i=0; i<${num_symlinks}; i++ )); do symlink="${run_dir}/$symlink" target="$FIXam/$target" - if [ -f "${target}" ]; then - ln_vrfy -sf ${relative_or_null} $target $symlink - else - print_err_msg_exit "\ - Cannot create symlink because target does not exist: - target = \"$target\"" - fi + create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" done # @@ -431,16 +416,34 @@ rm_vrfy -f time_stamp.out print_info_msg "$VERBOSE" " Creating links in the current run directory to cycle-independent model input files in the main experiment directory..." - -relative_or_null="" -if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then - relative_or_null="--relative" +# +# For experiments that are run in "community" mode, the model input files +# to which the symlinks will point are under the experiment directory. +# Thus, in this case, we use relative paths for the symlinks in order to +# keep the experiment directory more portable and the symlinks more readable. +# However, for experiments that are run in "nco" mode, the experiment +# directory in which the model input files are located is in general +# completely different than the run directory in which the symlinks will +# be created. Thus, in this case, there isn't really an advantage to +# using relative symlinks, so we use symlinks with absolute paths. +# +if [ "${RUN_ENVIR}" != "nco" ]; then + relative_link_flag="TRUE" +else + relative_link_flag="FALSE" fi -ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${run_dir} -ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${run_dir} -ln_vrfy -sf ${relative_or_null} ${NEMS_CONFIG_FP} ${run_dir} +create_symlink_to_file target="${DATA_TABLE_FP}" \ + symlink="${run_dir}/${DATA_TABLE_FN}" \ + relative="${relative_link_flag}" + +create_symlink_to_file target="${FIELD_TABLE_FP}" \ + symlink="${run_dir}/${FIELD_TABLE_FN}" \ + relative="${relative_link_flag}" +create_symlink_to_file target="${NEMS_CONFIG_FP}" \ + symlink="${run_dir}/${NEMS_CONFIG_FN}" \ + relative="${relative_link_flag}" if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat @@ -469,7 +472,9 @@ cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" run_dir = \"${run_dir}\"" else - ln_vrfy -sf ${relative_or_null} ${FV3_NML_FP} ${run_dir} + create_symlink_to_file target="${FV3_NML_FP}" \ + symlink="${run_dir}/${FV3_NML_FN}" \ + relative="${relative_link_flag}" fi # #----------------------------------------------------------------------- @@ -515,13 +520,9 @@ Call to function to create a diag table file for the current cycle's #----------------------------------------------------------------------- # if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - if [ "${MACHINE}" = "WCOSS_CRAY" ]; then - relative_or_null="" - else - relative_or_null="--relative" - fi - diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" - ln_vrfy -sf ${relative_or_null} ${diag_table_fp} ${run_dir} + create_symlink_to_file target="${cycle_dir}/${DIAG_TABLE_FN}" \ + symlink="${run_dir}/${DIAG_TABLE_FN}" \ + relative="${relative_link_flag}" fi # #----------------------------------------------------------------------- @@ -538,8 +539,13 @@ $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # -# Only for inline post, create the directory where post-processing output -# are stored (postprd_dir) +#----------------------------------------------------------------------- +# +# If doing inline post, create the directory in which the post-processing +# output will be stored (postprd_dir). +# +#----------------------------------------------------------------------- +# if [ ${WRITE_DOPOST} = "TRUE" ]; then yyyymmdd=${cdate:0:8} @@ -576,6 +582,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done done + fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 9d4ba1c090..87d454813b 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -92,7 +92,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} # #----------------------------------------------------------------------- # -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") @@ -326,9 +326,9 @@ post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${tmmark}.grib2" # For convenience, change location to postprd_dir (where the final output # from UPP will be located). Then loop through the two files that UPP # generates (i.e. "...bgdawp..." and "...bgrd3d..." files) and move, -# rename, and create links to them. +# rename, and create symlinks to them. # -cd_vrfy ${postprd_dir} +cd_vrfy "${postprd_dir}" basetime=$( date --date "$yyyymmdd $hh" +%y%j%H%M ) symlink_suffix="_${basetime}f${fhr}${post_mn}" fids=( "bgdawp" "bgrd3d" ) @@ -337,7 +337,9 @@ for fid in "${fids[@]}"; do post_orig_fn="${FID}.${post_fn_suffix}" post_renamed_fn="${NET}.t${cyc}z.${fid}${post_renamed_fn_suffix}" mv_vrfy ${tmp_dir}/${post_orig_fn} ${post_renamed_fn} - ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} + create_symlink_to_file target="${post_renamed_fn}" \ + symlink="${FID}${symlink_suffix}" \ + relative="TRUE" done rm_vrfy -rf ${tmp_dir} diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh new file mode 100644 index 0000000000..e54bf0a36a --- /dev/null +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -0,0 +1,163 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that is used to create a symbolic link +# ("symlink") to the specified target file ("target"). It checks for +# the existence of the target file and fails (with an appropriate error +# message) if that target does not exist or is not a file. Also, the +# argument "relative" determines whether a relative or an absolute path +# to the symlink is used. Note that on some platforms, relative symlinks +# are not supported. In those cases, an absolute path is used regardless +# of the setting of "relative". +# +#----------------------------------------------------------------------- +# +function create_symlink_to_file() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ +"target" \ +"symlink" \ +"relative" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Verify that the required arguments to this function have been specified. +# If not, print out an error message and exit. +# +#----------------------------------------------------------------------- +# + if [ -z "${target}" ]; then + print_err_msg_exit "\ +The argument \"target\" specifying the target of the symbolic link that +this function will create was not specified in the call to this function: + target = \"$target\"" + fi + + if [ -z "${symlink}" ]; then + print_err_msg_exit "\ +The argument \"symlink\" specifying the symbolic link that this function +will create was not specified in the call to this function: + symlink = \"$symlink\"" + fi +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local valid_vals_relative \ + relative_flag +# +#----------------------------------------------------------------------- +# +# If "relative" is not set (i.e. if it is set to a null string), reset +# it to a default value of "TRUE". Then check that it is set to a vaild +# value. +# +#----------------------------------------------------------------------- +# + relative=${relative:-"TRUE"} + + valid_vals_relative=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") + check_var_valid_value "relative" "valid_vals_relative" +# +#----------------------------------------------------------------------- +# +# Make sure that the target file exists and is a file. +# +#----------------------------------------------------------------------- +# + if [ ! -f "${target}" ]; then + print_err_msg_exit "\ +Cannot create symlink to specified target file because the latter does +not exist or is not a file: + target = \"$target\"" + fi +# +#----------------------------------------------------------------------- +# +# Set the flag that specifies whether or not a relative symlink should +# be created. +# +#----------------------------------------------------------------------- +# + relative_flag="" + if [ "${relative}" = "TRUE" ]; then + relative_flag="${RELATIVE_LINK_FLAG}" + fi +# +#----------------------------------------------------------------------- +# +# Create the symlink. +# +# Important note: +# In the ln_vrfy command below, do not quote ${relative_flag} because if +# is quoted (either single or double quotes) but happens to be a null +# string, it will be treated as the (empty) name of (or path to) the +# target and will cause an error. +# +#----------------------------------------------------------------------- +# + ln_vrfy -sf ${relative_flag} "$target" "$symlink" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index d11ec4720e..d73da5ef27 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -2,10 +2,10 @@ #----------------------------------------------------------------------- # # This is a generic function that executes the specified command (e.g. -# "cp", "mv", etc) with the specified options/arguments and then veri- -# fies that the command executed without errors. The first argument to -# this function is the command to execute while the remaining ones are -# the options/arguments to be passed to that command. +# "cp", "mv", etc) with the specified options/arguments and then verifies +# that the command executed without errors. The first argument to this +# function is the command to execute while the remaining ones are the +# options/arguments to be passed to that command. # #----------------------------------------------------------------------- # @@ -49,20 +49,19 @@ function filesys_cmd_vrfy() { # that function. # 2) If the caller is a sourced script, caller_name will be set to # "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. +# script since by defintion, it is sourced by another script or +# function. # 3) If the caller is the top-level script, caller_name will be set to # "main". # # Thus, if caller_name is set to "script" or "main", the caller is a # script, and if it is set to anything else, the caller is a function. # -# Below, the index into FUNCNAME and BASH_SOURCE is 2 (not 1 as is usu- -# ally the case) because this function is called by functions such as -# cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, and cd_vrfy, but these -# are just wrappers, and in the error and informational messages, we are -# really interested in the scripts/functions that in turn call these -# wrappers. +# Below, the index into FUNCNAME and BASH_SOURCE is 2 (not 1 as is usually +# the case) because this function is called by functions such as cp_vrfy, +# mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, and cd_vrfy, but these are just +# wrappers, and in the error and informational messages, we are really +# interested in the scripts/functions that in turn call these wrappers. # #----------------------------------------------------------------------- # @@ -73,6 +72,18 @@ function filesys_cmd_vrfy() { # #----------------------------------------------------------------------- # +# Declare local variables that are used later below. +# +#----------------------------------------------------------------------- +# + local cmd \ + output \ + exit_code \ + double_space \ + script_or_function +# +#----------------------------------------------------------------------- +# # Check that at least one argument is supplied. # #----------------------------------------------------------------------- @@ -105,7 +116,7 @@ are zero or more options and arguments to pass to that command. # #----------------------------------------------------------------------- # - local cmd="$1" + cmd="$1" shift # #----------------------------------------------------------------------- @@ -210,16 +221,15 @@ $output" # #----------------------------------------------------------------------- # -# The following are functions are counterparts of common filesystem com- -# mands "with verification", i.e. they execute a filesystem command -# (such as "cp" and "mv") and then verify that the execution was suc- -# cessful. +# The following are functions are counterparts of common filesystem +# commands "with verification", i.e. they execute a filesystem command +# (such as "cp" and "mv") and then verify that the execution was successful. # -# These functions are called using the "filesys_cmd_vrfy" function de- -# fined above. In each of these functions, we: +# These functions are called using the "filesys_cmd_vrfy" function defined +# above. In each of these functions, we: # -# 1) Save current shell options (in a global array) and then set new op- -# tions for this script/function. +# 1) Save current shell options (in a global array) and then set new +# options for this script/function. # 2) Call the generic function "filesys_cmd_vrfy" with the command of # interest (e.g. "cp") as the first argument and the arguments passed # in as the rest. diff --git a/ush/constants.sh b/ush/constants.sh index 89f38bc96a..fb19e61f84 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -1,11 +1,17 @@ -# This file is always sourced by another script (i.e. it's never run in -# its own shell), so there's no need to put the #!/bin/some_shell on the -# first line. +# +#----------------------------------------------------------------------- +# +# Mathematical and physical constants. +# +#----------------------------------------------------------------------- +# # Pi. pi_geom="3.14159265358979323846264338327" + # Degrees per radian. degs_per_radian=$( bc -l <<< "360.0/(2.0*$pi_geom)" ) + # Radius of the Earth in meters. -radius_Earth="6371000.0" +radius_Earth="6371200.0" diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 56e22b5a67..10ad396ce9 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -66,8 +66,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - local i \ - diag_table_fp \ + local diag_table_fp \ settings # #----------------------------------------------------------------------- @@ -82,8 +81,8 @@ run directory... run_dir = \"${run_dir}\"" - diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" - print_info_msg "$VERBOSE" " + diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" + print_info_msg "$VERBOSE" " Using the template diagnostics table file: @@ -93,11 +92,11 @@ to create: diag_table_fp = \"${diag_table_fp}\"" - settings=" + settings=" starttime: !datetime ${CDATE} cres: ${CRES}" - $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ + $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ print_err_msg_exit " !!!!!!!!!!!!!!!!! @@ -105,7 +104,6 @@ fill_jinja_template.py failed! !!!!!!!!!!!!!!!!! " - # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 1269db0ab6..260c37f46d 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -394,7 +394,9 @@ Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): EXPTDIR = \"${EXPTDIR}\" WFLOW_LAUNCH_SCRIPT_FP = \"${WFLOW_LAUNCH_SCRIPT_FP}\"" -ln_vrfy -fs "${WFLOW_LAUNCH_SCRIPT_FP}" "$EXPTDIR" +create_symlink_to_file target="${WFLOW_LAUNCH_SCRIPT_FP}" \ + symlink="${EXPTDIR}/${WFLOW_LAUNCH_SCRIPT_FN}" \ + relative="FALSE" # #----------------------------------------------------------------------- # @@ -819,7 +821,7 @@ The experiment directory is: > EXPTDIR=\"$EXPTDIR\" " -case $MACHINE in +case "$MACHINE" in "CHEYENNE") print_info_msg "\ diff --git a/ush/link_fix.sh b/ush/link_fix.sh index 29ce0ecc1d..e9a641d962 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -83,7 +83,7 @@ function link_fix() { fp_prev \ fp \ fn \ - relative_or_null \ + relative_link_flag \ cres \ tmp \ fns_sfc_climo_with_halo_in_fn \ @@ -356,24 +356,32 @@ Please ensure that all files have the same resolution." # #----------------------------------------------------------------------- # - relative_or_null="" - if [ "${run_task}" = "TRUE" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then - relative_or_null="--relative" +# If the task in consideration (which will be one of the pre-processing +# tasks MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN) was run, then +# the target files will be located under the experiment directory. In +# this case, we use relative symlinks in order the experiment directory +# more portable and the symlinks more readable. However, if the task +# was not run, then pregenerated grid, orography, or surface climatology +# files will be used, and those will be located in an arbitrary directory +# (specified by the user) that is somwehere outside the experiment +# directory. Thus, in this case, there isn't really an advantage to using +# relative symlinks, so we use symlinks with absolute paths. +# + if [ "${run_task}" = "TRUE" ]; then + relative_link_flag="TRUE" + else + relative_link_flag="FALSE" fi for fp in "${fps[@]}"; do - if [ -f "$fp" ]; then - ln_vrfy -sf ${relative_or_null} $fp . - else - print_err_msg_exit "\ -Cannot create symlink because target file (fp) does not exist: - fp = \"${fp}\"" - fi + fn=$( basename $fp ) + create_symlink_to_file target="$fp" symlink="$fn" \ + relative="${relative_link_flag}" done # #----------------------------------------------------------------------- # -# Set the C-resolution based on the resolution appearing in the file +# Set the C-resolution based on the resolution appearing in the file # names. # #----------------------------------------------------------------------- @@ -383,8 +391,8 @@ Cannot create symlink because target file (fp) does not exist: #----------------------------------------------------------------------- # # If considering grid files, create a symlink to the halo4 grid file -# that does not contain the halo size in its name. This is needed by -# the tasks that generate the initial and lateral boundary condition +# that does not contain the halo size in its name. This is needed by +# the tasks that generate the initial and lateral boundary condition # files. # #----------------------------------------------------------------------- @@ -393,52 +401,33 @@ Cannot create symlink because target file (fp) does not exist: target="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" symlink="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" - if [ -f "${target}" ]; then - ln_vrfy -sf $target $symlink - else - print_err_msg_exit "\ -Cannot create symlink because the target file (target) in the directory -specified by FIXLAM does not exist: - FIXLAM = \"${FIXLAM}\" - target = \"${target}\"" - fi + create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" # -# The surface climatology file generation code looks for a grid file +# The surface climatology file generation code looks for a grid file # having a name of the form "C${GFDLgrid_RES}_grid.tile7.halo4.nc" (i.e. -# the resolution used in this file is that of the number of grid points -# per horizontal direction per tile, just like in the global model). -# Thus, if we are running this code, if the grid is of GFDLgrid type, and -# if we are not using GFDLgrid_RES in filenames (i.e. we are using the -# equivalent global uniform grid resolution instead), then create a link -# whose name uses the GFDLgrid_RES that points to the link whose name uses -# the equivalent global uniform resolution. +# the C-resolution used in the name of this file is the number of grid +# points per horizontal direction per tile, just like in the global model). +# Thus, if we are running the MAKE_SFC_CLIMO_TN task, if the grid is of +# GFDLgrid type, and if we are not using GFDLgrid_RES in filenames (i.e. +# we are using the equivalent global uniform grid resolution instead), +# then create a link whose name uses the GFDLgrid_RES that points to the +# link whose name uses the equivalent global uniform resolution. # if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] && \ [ "${GRID_GEN_METHOD}" = "GFDLgrid" ] && \ [ "${GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES}" = "FALSE" ]; then - target="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" symlink="C${GFDLgrid_RES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" - if [ -f "${target}" ]; then - ln_vrfy -sf $target $symlink - else - print_err_msg_exit "\ -Cannot create symlink because the target file (target) in the directory -specified by FIXLAM does not exist: - FIXLAM = \"${FIXLAM}\" - target = \"${target}\"" - fi - + create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" fi fi # #----------------------------------------------------------------------- # -# If considering surface climatology files, create symlinks to the sur- -# face climatology files that do not contain the halo size in their -# names. These are needed by the task that generates the initial condi- -# tion files. +# If considering surface climatology files, create symlinks to the surface +# climatology files that do not contain the halo size in their names. +# These are needed by the task that generates the initial condition files. # #----------------------------------------------------------------------- # @@ -451,19 +440,13 @@ specified by FIXLAM does not exist: for (( i=0; i<${num_fields}; i++ )); do target="${fns_sfc_climo_with_halo_in_fn[$i]}" symlink="${fns_sfc_climo_no_halo_in_fn[$i]}" - if [ -f "$target" ]; then - ln_vrfy -sf $target $symlink - else - print_err_msg_exit "\ -Cannot create symlink because target file (target) does not exist: - target = \"${target}\"" - fi + create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" done # -# In order to be able to specify the surface climatology file names in +# In order to be able to specify the surface climatology file names in # the forecast model's namelist file, in the FIXLAM directory a symlink # must be created for each surface climatology field that has "tile1" in -# its name (and no "halo") and which points to the corresponding "tile7.halo0" +# its name (and no "halo") and which points to the corresponding "tile7.halo0" # file. # tmp=( "${SFC_CLIMO_FIELDS[@]/#/${cres}.}" ) @@ -473,13 +456,7 @@ Cannot create symlink because target file (target) does not exist: for (( i=0; i<${num_fields}; i++ )); do target="${fns_sfc_climo_tile7_halo0_in_fn[$i]}" symlink="${fns_sfc_climo_tile1_no_halo_in_fn[$i]}" - if [ -f "$target" ]; then - ln_vrfy -sf $target $symlink - else - print_err_msg_exit "\ -Cannot create symlink because target file (target) does not exist: - target = \"${target}\"" - fi + create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" done fi diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index 846f12827e..a6d4c6dfd1 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -84,23 +84,23 @@ function set_FV3nml_stoch_params() { # #----------------------------------------------------------------------- # -# For a given cycle and member, generate a namelist file with unique +# For a given cycle and member, generate a namelist file with unique # seed values. # #----------------------------------------------------------------------- # -ensmem_name="mem${ENSMEM_INDX}" + ensmem_name="mem${ENSMEM_INDX}" -fv3_nml_ensmem_fp="${CYCLE_BASEDIR}/${cdate}/${ensmem_name}/${FV3_NML_FN}" + fv3_nml_ensmem_fp="${CYCLE_BASEDIR}/${cdate}/${ensmem_name}/${FV3_NML_FN}" -ensmem_num=$((10#${ENSMEM_INDX})) + ensmem_num=$((10#${ENSMEM_INDX})) -iseed_shum=$(( cdate*1000 + ensmem_num*10 + 2 )) -iseed_skeb=$(( cdate*1000 + ensmem_num*10 + 3 )) -iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) -iseed_spp=$(( cdate*1000 + ensmem_num*10 + 4 )) + iseed_shum=$(( cdate*1000 + ensmem_num*10 + 2 )) + iseed_skeb=$(( cdate*1000 + ensmem_num*10 + 3 )) + iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) + iseed_spp=$(( cdate*1000 + ensmem_num*10 + 4 )) -settings="\ + settings="\ 'nam_stochy': { 'iseed_shum': ${iseed_shum}, 'iseed_skeb': ${iseed_skeb}, @@ -110,11 +110,11 @@ settings="\ 'iseed_spp': ${iseed_spp}, }" -$USHDIR/set_namelist.py -q \ - -n ${FV3_NML_FP} \ - -u "$settings" \ - -o ${fv3_nml_ensmem_fp} || \ - print_err_msg_exit "\ + $USHDIR/set_namelist.py -q \ + -n ${FV3_NML_FP} \ + -u "$settings" \ + -o ${fv3_nml_ensmem_fp} || \ + print_err_msg_exit "\ Call to python script set_namelist.py to set the variables in the FV3 namelist file that specify the paths to the surface climatology files failed. Parameters passed to this script are: diff --git a/ush/setup.sh b/ush/setup.sh index 1633e91efc..a3708cd932 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -469,9 +469,16 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" # several queues. These queues are defined in the default and local # workflow/experiment configuration script. # +# Also, set the machine-dependent flag RELAITVE_OR_NULL that specifies +# the flag to pass to the link creation command (ln_vrfy) when attempting +# to create relative symlinks. On machines that don't support relative +# symlinks, it should be set to a null string. +# #----------------------------------------------------------------------- # -case $MACHINE in +RELATIVE_LINK_FLAG="" + +case "$MACHINE" in "WCOSS_CRAY") NCORES_PER_NODE="24" @@ -479,6 +486,8 @@ case $MACHINE in QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} QUEUE_FCST=${QUEUE_FCST:-"dev"} +# + RELATIVE_LINK_FLAG="" ;; "WCOSS_DELL_P3") @@ -487,58 +496,70 @@ case $MACHINE in QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} QUEUE_FCST=${QUEUE_FCST:-"dev"} +# + RELATIVE_LINK_FLAG="--relative" ;; "HERA") NCORES_PER_NODE=40 - SCHED="${SCHED:-slurm}" + SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} PARTITION_FCST=${PARTITION_FCST:-"hera"} QUEUE_FCST=${QUEUE_FCST:-"batch"} +# + RELATIVE_LINK_FLAG="--relative" ;; "ORION") NCORES_PER_NODE=40 - SCHED="${SCHED:-slurm}" + SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} PARTITION_FCST=${PARTITION_FCST:-"orion"} QUEUE_FCST=${QUEUE_FCST:-"batch"} +# + RELATIVE_LINK_FLAG="--relative" ;; "JET") NCORES_PER_NODE=24 - SCHED="${SCHED:-slurm}" + SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} +# + RELATIVE_LINK_FLAG="--relative" ;; "ODIN") NCORES_PER_NODE=24 - SCHED="${SCHED:-slurm}" + SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} PARTITION_HPSS=${PARTITION_HPSS:-"workq"} QUEUE_HPSS=${QUEUE_HPSS:-"workq"} PARTITION_FCST=${PARTITION_FCST:-"workq"} QUEUE_FCST=${QUEUE_FCST:-"workq"} +# + RELATIVE_LINK_FLAG="--relative" ;; "CHEYENNE") NCORES_PER_NODE=36 - SCHED="${SCHED:-pbspro}" + SCHED=${SCHED:-"pbspro"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} QUEUE_HPSS=${QUEUE_HPSS:-"regular"} QUEUE_FCST=${QUEUE_FCST:-"regular"} +# + RELATIVE_LINK_FLAG="--relative" ;; "STAMPEDE") @@ -550,6 +571,8 @@ case $MACHINE in QUEUE_HPSS=${QUEUE_HPSS:-"normal"} PARTITION_FCST=${PARTITION_FCST:-"normal"} QUEUE_FCST=${QUEUE_FCST:-"normal"} +# + RELATIVE_LINK_FLAG="--relative" ;; esac @@ -821,7 +844,7 @@ VX_CONFIG_DIR="$TEMPLATE_DIR/parm" METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" MET_CONFIG="$TEMPLATE_DIR/parm/met" -case $MACHINE in +case "$MACHINE" in "WCOSS_CRAY") FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} @@ -1534,6 +1557,7 @@ elif [ "$DO_ENSEMBLE" = "FALSE" ] || \ [ "$DO_ENSEMBLE" = "NO" ]; then DO_ENSEMBLE="FALSE" fi + NDIGITS_ENSMEM_NAMES="0" ENSMEM_NAMES=("") FV3_NML_ENSMEM_FPS=("") @@ -2682,6 +2706,14 @@ THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" # #----------------------------------------------------------------------- # +# Flag for creating relative symlinks (as opposed to absolute ones). +# +#----------------------------------------------------------------------- +# +RELATIVE_LINK_FLAG="${RELATIVE_LINK_FLAG}" +# +#----------------------------------------------------------------------- +# # Parameters that indicate whether or not various parameterizations are # included in and called by the phsics suite. # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 1ebe07affc..bc6a2f0832 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -22,12 +22,20 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Set the directory in which the files defining the various utility -# functions are located. +# Set necessary directory variables. # #----------------------------------------------------------------------- # - local bashutils_dir="${scrfunc_dir}/bash_utils" + local ushdir="${scrfunc_dir}" + local bashutils_dir="${ushdir}/bash_utils" +# +#----------------------------------------------------------------------- +# +# Source the file containing workflow constants. +# +#----------------------------------------------------------------------- +# + . ${ushdir}/constants.sh # #----------------------------------------------------------------------- # @@ -172,6 +180,15 @@ function source_util_funcs() { #----------------------------------------------------------------------- # . ${bashutils_dir}/get_charvar_from_netcdf.sh +# +#----------------------------------------------------------------------- +# +# Source the file containing the function that creates a symlink to a +# file (including performing checks). +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/create_symlink_to_file.sh } source_util_funcs diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 2edcf6b56d..96998bc50b 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,3 +1,6 @@ +# +# Define valid values for various global experiment/workflow variables. +# valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE") @@ -67,5 +70,5 @@ valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_COMPILER=("intel" "gnu") -valid_vals_SUB_HOURLY_POST=("TRUE" "true" "FALSE" "false" "YES" "yes" "NO" "no") +valid_vals_SUB_HOURLY_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") From 1e1a41dfb1267f4247cf0774338000fd3752b14f Mon Sep 17 00:00:00 2001 From: Siwei He <43216838+siwei-noaa@users.noreply.github.com> Date: Tue, 13 Jul 2021 10:53:45 -0600 Subject: [PATCH 060/203] WE2E test for MET verification. (#537) * WE2E test for MET validation. * Add a header to the config file. * Update config.MET_validation.sh * Change 'validation' to 'verification' Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> --- .../config.MET_verification.sh | 37 +++++++++++++++++++ tests/baselines_list.txt | 1 + 2 files changed, 38 insertions(+) create mode 100644 tests/baseline_configs/config.MET_verification.sh diff --git a/tests/baseline_configs/config.MET_verification.sh b/tests/baseline_configs/config.MET_verification.sh new file mode 100644 index 0000000000..33d035ebef --- /dev/null +++ b/tests/baseline_configs/config.MET_verification.sh @@ -0,0 +1,37 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully with MET verification. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" +FCST_LEN_HRS="36" +LBC_SPEC_INTVL_HRS="6" + +DATE_FIRST_CYCL="20190415" +DATE_LAST_CYCL="20190415" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" + +WTIME_RUN_FCST="01:00:00" + +MODEL="FV3_GFS_v15p2_CONUS_25km" +METPLUS_PATH="/contrib/METplus/METplus-4.0.0" +MET_INSTALL_DIR="/contrib/met/10.0.0" + +RUN_TASK_VX_GRIDSTAT="TRUE" +RUN_TASK_VX_POINTSTAT="TRUE" + + +USE_USER_STAGED_EXTRN_FILES="TRUE" + diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index f920d9a667..5229c95a90 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -82,3 +82,4 @@ suite_FV3_GSD_SAR suite_FV3_GSD_v0 suite_FV3_HRRR suite_FV3_RRFS_v1beta +MET_verification From 61c76d8a66baabd9ad9d1bb59862f11c9c5a17a9 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:53:45 -0400 Subject: [PATCH 061/203] Modify script for 3 digit forecast length (#549) --- scripts/exregional_run_fcst.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 89caf2e51a..1aa4ade8c9 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -564,15 +564,22 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cd_vrfy ${postprd_dir} - for fhr in $(seq -f "%02g" 0 ${FCST_LEN_HRS}); do - post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) + for fhr in $(seq -f "%03g" 0 ${FCST_LEN_HRS}); do + + if [ ${fhr:0:1} = "0" ]; then + fhr_d=${fhr:1:2} + else + fhr_d=${fhr} + fi + + post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr_d} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) post_mn=${post_time:10:2} post_mn_or_null="" - post_fn_suffix="GrbF${fhr}" - post_renamed_fn_suffix="f0${fhr}${post_mn_or_null}.${tmmark}.grib2" + post_fn_suffix="GrbF${fhr_d}" + post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${tmmark}.grib2" basetime=$( date --date "$yyyymmdd $hh" +%y%j%H%M ) - symlink_suffix="_${basetime}f0${fhr}${post_mn}" + symlink_suffix="_${basetime}f${fhr}${post_mn}" fids=( "bgdawp" "bgrd3d" ) for fid in "${fids[@]}"; do FID="${fid^^}" From 3e471ed960b70a64c28722b4e99d14e0f87f660e Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Thu, 15 Jul 2021 12:53:14 -0400 Subject: [PATCH 062/203] Update syntax for latest version of lmod. (#542) ## DESCRIPTION OF CHANGES: This PR addresses an error that occurs at run time ```ModuleNotFoundError: No module named 'f90nml'```. A similar error may appear in any of the tasks that use a conda environment, and could potentially list `jinja2` instead. The version of lmod on Hera was updated. The behavior of the `system` command has changed and now runs only in a subshell. As an lmod-version-independent fix, the module will no longer activate a conda environment, but will set the conda environment that should be activated through an environment variable, then the script will activate any required conda environment. Note: Users who do not run with Rocoto, but still use these modules, will have an extra step of activating the conda environment manually. ## TESTS CONDUCTED: Built and ran a single WE2E test (grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2) successfully on Hera with these modifications, as well as a WE2E test (DOT_OR_USCORE) on Orion as a sanity check. --- modulefiles/tasks/hera/make_grid.local | 5 ++--- modulefiles/tasks/hera/make_ics.local | 5 ++--- modulefiles/tasks/hera/make_lbcs.local | 5 ++--- modulefiles/tasks/hera/run_fcst.local | 5 ++--- modulefiles/tasks/jet/make_grid.local | 5 ++--- modulefiles/tasks/jet/make_ics.local | 5 ++--- modulefiles/tasks/jet/make_lbcs.local | 5 ++--- modulefiles/tasks/jet/run_fcst.local | 5 ++--- modulefiles/tasks/orion/make_grid.local | 5 ++--- modulefiles/tasks/orion/make_ics.local | 5 ++--- modulefiles/tasks/orion/make_lbcs.local | 5 ++--- modulefiles/tasks/orion/run_fcst.local | 5 ++--- ush/load_modules_run_task.sh | 11 +++++++++++ 13 files changed, 35 insertions(+), 36 deletions(-) diff --git a/modulefiles/tasks/hera/make_grid.local b/modulefiles/tasks/hera/make_grid.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/hera/make_grid.local +++ b/modulefiles/tasks/hera/make_grid.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/hera/make_ics.local b/modulefiles/tasks/hera/make_ics.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/hera/make_ics.local +++ b/modulefiles/tasks/hera/make_ics.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/hera/make_lbcs.local b/modulefiles/tasks/hera/make_lbcs.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/hera/make_lbcs.local +++ b/modulefiles/tasks/hera/make_lbcs.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/hera/run_fcst.local b/modulefiles/tasks/hera/run_fcst.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/hera/run_fcst.local +++ b/modulefiles/tasks/hera/run_fcst.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/make_grid.local b/modulefiles/tasks/jet/make_grid.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/jet/make_grid.local +++ b/modulefiles/tasks/jet/make_grid.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/make_ics.local b/modulefiles/tasks/jet/make_ics.local index d03fe381e9..c3c0f61e55 100644 --- a/modulefiles/tasks/jet/make_ics.local +++ b/modulefiles/tasks/jet/make_ics.local @@ -3,6 +3,5 @@ module load wgrib2/2.0.8 module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/make_lbcs.local b/modulefiles/tasks/jet/make_lbcs.local index d03fe381e9..c3c0f61e55 100644 --- a/modulefiles/tasks/jet/make_lbcs.local +++ b/modulefiles/tasks/jet/make_lbcs.local @@ -3,6 +3,5 @@ module load wgrib2/2.0.8 module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/run_fcst.local b/modulefiles/tasks/jet/run_fcst.local index 52955b476c..011a832c9b 100644 --- a/modulefiles/tasks/jet/run_fcst.local +++ b/modulefiles/tasks/jet/run_fcst.local @@ -1,6 +1,5 @@ #%Module module use -a /contrib/miniconda3/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 1dd1c7e50d..93a76f6ca0 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,6 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 1dd1c7e50d..93a76f6ca0 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,6 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 1dd1c7e50d..93a76f6ca0 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,6 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 1dd1c7e50d..93a76f6ca0 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,6 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 -if [module-info mode load] { - system "conda activate regional_workflow" -} + +setenv SRW_ENV regional_workflow diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index c2e7cbc9a9..862bf9338c 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -321,6 +321,17 @@ ules_dir) for the specified task (task_name) failed: module list #fi #End if statement for tasks that load no modules + +# Modules that use conda and need an environment activated will set the +# SRW_ENV variable to the name of the environment to be activated. That +# must be done within the script, and not inside the module. Do that +# now. + +if [ -n "${SRW_ENV:-}" ] ; then + conda activate ${SRW_ENV} +fi + + # #----------------------------------------------------------------------- # From e87c6e626c793b8e85d6aaf37f7356cb7559e3e8 Mon Sep 17 00:00:00 2001 From: Jili Dong Date: Thu, 15 Jul 2021 13:54:30 -0400 Subject: [PATCH 063/203] change input.nml for filter topo to fix task failure (#547) ## DESCRIPTION OF CHANGES: Remove "refine_ratio = ${refine_ratio}" in scripts/exregional_make_orog.sh to fix filter_topo failure ## TESTS CONDUCTED: regional ESG grids/topography ## DEPENDENCIES: ## DOCUMENTATION: ## ISSUE (optional): Fixes issue mentioned in #546 --- scripts/exregional_make_orog.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index dfd77bc7c3..4930295e7b 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -493,7 +493,6 @@ cat > "${filter_dir}/input.nml" < Date: Thu, 15 Jul 2021 20:47:09 -0600 Subject: [PATCH 064/203] Copy and link NEMS field dictionary file (#545) Copy and link NEMS field dictionary file: * Copy field dictionary file from tests/parm into exptdir * Link field dictionary file into run_dir during fcst --- scripts/exregional_run_fcst.sh | 4 ++++ ush/generate_FV3LAM_wflow.sh | 9 +++++++++ ush/setup.sh | 27 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 1aa4ade8c9..2da489ee58 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -445,6 +445,10 @@ create_symlink_to_file target="${NEMS_CONFIG_FP}" \ symlink="${run_dir}/${NEMS_CONFIG_FN}" \ relative="${relative_link_flag}" +create_symlink_to_file target="${FIELD_DICT_FP}" \ + symlink="${run_dir}/${FIELD_DICT_FN}" \ + relative="${relative_link_flag}" + if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 260c37f46d..8f4bbdf3ca 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -558,6 +558,15 @@ Copying the CCPP physics suite definition XML file from its location in the forecast model directory sturcture to the experiment directory..." cp_vrfy "${CCPP_PHYS_SUITE_IN_CCPP_FP}" "${CCPP_PHYS_SUITE_FP}" # +# Copy the field dictionary file from its location in the +# clone of the FV3 code repository to the experiment directory (EXPT- +# DIR). +# +print_info_msg "$VERBOSE" " +Copying the field dictionary file from its location in the forecast +model directory sturcture to the experiment directory..." +cp_vrfy "${FIELD_DICT_IN_UWM_FP}" "${FIELD_DICT_FP}" +# #----------------------------------------------------------------------- # # Set parameters in the FV3-LAM namelist file. diff --git a/ush/setup.sh b/ush/setup.sh index a3708cd932..06c8289242 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1445,6 +1445,29 @@ fi # #----------------------------------------------------------------------- # +# Set: +# +# 1) the variable FIELD_DICT_FN to the name of the field dictionary +# file. +# 2) the variable FIELD_DICT_IN_UWM_FP to the full path of this +# file in the forecast model's directory structure. +# 3) the variable FIELD_DICT_FP to the full path of this file in +# the experiment directory. +# +#----------------------------------------------------------------------- +# +FIELD_DICT_FN="fd_nems.yaml" +FIELD_DICT_IN_UWM_FP="${UFS_WTHR_MDL_DIR}/tests/parm/${FIELD_DICT_FN}" +FIELD_DICT_FP="${EXPTDIR}/${FIELD_DICT_FN}" +if [ ! -f "${FIELD_DICT_IN_UWM_FP}" ]; then + print_err_msg_exit "\ +The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist +in the local clone of the ufs-weather-model: + FIELD_DICT_IN_UWM_FP = \"${FIELD_DICT_IN_UWM_FP}\"" +fi +# +#----------------------------------------------------------------------- +# # Call the function that sets the ozone parameterization being used and # modifies associated parameters accordingly. # @@ -2691,6 +2714,10 @@ CCPP_PHYS_SUITE_FN="${CCPP_PHYS_SUITE_FN}" CCPP_PHYS_SUITE_IN_CCPP_FP="${CCPP_PHYS_SUITE_IN_CCPP_FP}" CCPP_PHYS_SUITE_FP="${CCPP_PHYS_SUITE_FP}" +FIELD_DICT_FN="${FIELD_DICT_FN}" +FIELD_DICT_IN_UWM_FP="${FIELD_DICT_IN_UWM_FP}" +FIELD_DICT_FP="${FIELD_DICT_FP}" + DATA_TABLE_FP="${DATA_TABLE_FP}" FIELD_TABLE_FP="${FIELD_TABLE_FP}" FV3_NML_FN="${FV3_NML_FN}" # This may not be necessary... From bbfcfa34ea921a4006753b88986a5c6e4714068c Mon Sep 17 00:00:00 2001 From: Siwei He <43216838+siwei-noaa@users.noreply.github.com> Date: Fri, 16 Jul 2021 16:20:10 -0600 Subject: [PATCH 065/203] Feature/met on hera (#552) ## DESCRIPTION OF CHANGES: 1. Add if condition in tests/run_experiments.sh so to get informative error message when MET and MET paths are not available on a machine. 2. Remove MET/MET+ paths in tests/baseline_configs/config.verification.sh ## TESTS CONDUCTED: A test run was conducted on Hera and has been finished successfully. Another test was run on Jet, and it failed in the expected way. ## DEPENDENCIES: To have MET verification run successfully, the observational data (e.g., CCPA, MRMS, NDAS) must be available. ## DOCUMENTATION: N/A ## ISSUE (optional): This is a follow up PR to complete the previous one in https://github.com/NOAA-EMC/regional_workflow/pull/537 ## CONTRIBUTORS (optional): @gsketefian contributed the revision. --- .../config.MET_verification.sh | 2 -- tests/run_experiments.sh | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/baseline_configs/config.MET_verification.sh b/tests/baseline_configs/config.MET_verification.sh index 33d035ebef..fd5ce114c3 100644 --- a/tests/baseline_configs/config.MET_verification.sh +++ b/tests/baseline_configs/config.MET_verification.sh @@ -26,8 +26,6 @@ EXTRN_MDL_NAME_LBCS="FV3GFS" WTIME_RUN_FCST="01:00:00" MODEL="FV3_GFS_v15p2_CONUS_25km" -METPLUS_PATH="/contrib/METplus/METplus-4.0.0" -MET_INSTALL_DIR="/contrib/met/10.0.0" RUN_TASK_VX_GRIDSTAT="TRUE" RUN_TASK_VX_POINTSTAT="TRUE" diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 00aca3d30b..2eb82965ac 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -779,6 +779,33 @@ EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" fi + +# +#----------------------------------------------------------------------- +# +# Set MET and MET+ paths, if necessary. +# +#----------------------------------------------------------------------- +# + if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then + if [ "$MACHINE" = "HERA" ]; then + met_install_dir="/contrib/met/10.0.0" + metplus_path="/contrib/METplus/METplus-4.0.0" + else + print_err_msg_exit "\ +The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) have not been specified for +this machine (MACHINE): MACHINE= \"${MACHINE}\" " + fi + + str=${str}" +# +# MET and MET+ paths. +# +METPLUS_PATH=\"${metplus_path}\" +MET_INSTALL_DIR=\"${met_install_dir}\"" + + fi + # #----------------------------------------------------------------------- # From bb28b5fd2174faa7d55aa8b597b79eb265a50e86 Mon Sep 17 00:00:00 2001 From: JulieSchramm Date: Wed, 21 Jul 2021 12:52:48 -0600 Subject: [PATCH 066/203] Modifications to allow workflow to run to completion with GNU build on Hera (#526) * Modifications to scripts to allow the workflow to run to completion with the GNU build on Hera. The python/miniconda3 module must be unloaded prior to running the executable. * "module" is not the best name for a variable, changed to a_module * Address comments: - Clarify message about why modules are being unloaded - Remove print message if nothing is unloaded to reduce clutter in log file - Rename "module_to_unload" to "modules_to_unload" - Change "a_module" to "module_to_unload" for clarity * Clarify purpose of the unload_python.sh function. * Add the "why" --- scripts/exregional_make_ics.sh | 7 +++ scripts/exregional_make_lbcs.sh | 7 +++ scripts/exregional_run_fcst.sh | 7 +++ ush/bash_utils/unload_python.sh | 97 +++++++++++++++++++++++++++++++++ ush/source_util_funcs.sh | 6 ++ 5 files changed, 124 insertions(+) create mode 100644 ush/bash_utils/unload_python.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a2c035eaea..a8efca308e 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -615,6 +615,13 @@ $settings" # of chgres_cube is nonzero. # A similar thing happens in the forecast task. # +#----------------------------------------------------------------------- +# Deactivate the conda environment to avoid conflicts between the +# environment used by the regional workflow (prepended to your $PATH) +# and that used in the run environment. +#----------------------------------------------------------------------- +unload_python + ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index de112f7e9a..ef1c683f36 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -516,6 +516,13 @@ $settings" # of chgres_cube is nonzero. # A similar thing happens in the forecast task. # +#----------------------------------------------------------------------- +# Deactivate the conda environment to avoid conflicts between the +# environment used by the regional workflow (prepended to your $PATH) +# and that used in the run environment. +#----------------------------------------------------------------------- + unload_python + ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 2da489ee58..2ab80320c6 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -539,6 +539,13 @@ fi # #----------------------------------------------------------------------- # +#----------------------------------------------------------------------- +# Deactivate the conda environment to avoid conflicts between the +# environment used by the regional workflow (prepended to your $PATH) +# and that used in the run environment. +#----------------------------------------------------------------------- +unload_python + $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." diff --git a/ush/bash_utils/unload_python.sh b/ush/bash_utils/unload_python.sh new file mode 100644 index 0000000000..f5879de0c0 --- /dev/null +++ b/ush/bash_utils/unload_python.sh @@ -0,0 +1,97 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that detects if a python or miniconds3 +# module is loaded, and if so, unloads that module. This may be +# necessary on machines where the loaded python environment isn't +# compatible with environment set for the compiler. +# +#----------------------------------------------------------------------- +# +function unload_python() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Check arguments. +# +#----------------------------------------------------------------------- +# + if [ "$#" -ne 0 ]; then + + print_err_msg_exit " +Incorrect number of arguments specified: + + Function name: \"${func_name}\" + Number of arguments specified: $# + +Usage: + + ${func_name} + +" + + fi +# +#----------------------------------------------------------------------- +# +# If the miniconda or python modules are loaded, unload them +# +#----------------------------------------------------------------------- +# + + modules_to_unload=( python miniconda3 ) + loaded_modules=$(module list 2>&1) + + for module_to_unload in ${modules_to_unload[@]}; do + if [[ "${loaded_modules}" =~ "${module_to_unload}" ]]; then + print_info_msg "\ +Unloading module ${module_to_unload} libraries needed for workflow generation but not for running the workflow... " + module unload ${module_to_unload} + fi + done + + loaded_modules=$(module list 2>&1) + print_info_msg "\ +Loaded modules are: $loaded_modules " + +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index bc6a2f0832..d8b866b69d 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -183,6 +183,12 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # +# Unload python or miniconda3 if loaded +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/unload_python.sh +# # Source the file containing the function that creates a symlink to a # file (including performing checks). # From 9bd3926f8d17be2c62344d7faf041c8eb064532b Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 22 Jul 2021 14:21:23 -0600 Subject: [PATCH 067/203] Remove symbolic linking of diag_table when running in ensemble mode --- scripts/exregional_run_fcst.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 2ab80320c6..9750dee701 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -515,22 +515,6 @@ Call to function to create a diag table file for the current cycle's # #----------------------------------------------------------------------- # -# If running ensemble forecasts, create a link to the cycle-specific -# diagnostic tables file in the cycle directory. Note that this link -# should not be made if not running ensemble forecasts because in that -# case, the cycle directory is the run directory (and we would be creating -# a symlink with the name of a file that already exists). -# -#----------------------------------------------------------------------- -# -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - create_symlink_to_file target="${cycle_dir}/${DIAG_TABLE_FN}" \ - symlink="${run_dir}/${DIAG_TABLE_FN}" \ - relative="${relative_link_flag}" -fi -# -#----------------------------------------------------------------------- -# # Run the FV3-LAM model. Note that we have to launch the forecast from # the current cycle's directory because the FV3 executable will look for # input files in the current directory. Since those files have been From a7dea1374056cd1a5d977631bf8facbd635a83a4 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 22 Jul 2021 14:48:30 -0600 Subject: [PATCH 068/203] Enable fetching of (and running with) FV3GFS external model data in netcdf format from NOAA HPSS (#555) ## DESCRIPTION OF CHANGES: This PR adds the capability to fetch FV3GFS external model data in netcdf format from NOAA HPSS. Main changes: 1) Modify workflow scripts where necessary to include stanzas for FV3GFS_FILE_FMT_ICS and FV3GFS_FMT_LBCS being set to "netcdf" (currently, only "nemsio" and "grib2" are valid values for these parameters). 2) Add a WE2E test for this capability (config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh). ## TESTS CONDUCTED: Successfully ran the following 3 WE2E tests (including the new one introduced here) on Hera: 1) get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100 2) get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100 3) get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000 ## ISSUE (optional): This resolves Issue #470. --- scripts/exregional_make_ics.sh | 11 +- scripts/exregional_make_lbcs.sh | 9 +- ...V3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh | 19 +++ tests/baselines_list.txt | 1 + ush/get_extrn_mdl_file_dir_info.sh | 111 +++++------------- 5 files changed, 62 insertions(+), 89 deletions(-) create mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a8efca308e..98a82d815e 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -390,7 +390,6 @@ case "${EXTRN_MDL_NAME_ICS}" in "GSMGFS") external_model="GSMGFS" - fn_atm="${EXTRN_MDL_FNS[0]}" fn_sfc="${EXTRN_MDL_FNS[1]}" input_type="gfs_gaussian_nemsio" # For spectral GFS Gaussian grid in nemsio format. @@ -408,31 +407,25 @@ case "${EXTRN_MDL_NAME_ICS}" in "FV3GFS") if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then external_model="FV3GFS" + input_type="gaussian_nemsio" # For FV3GFS data on a Gaussian grid in nemsio format. tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - fn_atm="${EXTRN_MDL_FNS[0]}" fn_sfc="${EXTRN_MDL_FNS[1]}" - input_type="gaussian_nemsio" # For FV3-GFS Gaussian grid in nemsio format. convert_nst=True elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then external_model="GFS" fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" convert_nst=False - elif [ "${FV3GFS_FILE_FMT_ICS}" = "netcdf" ]; then - external_model="FV3GFS" - + input_type="gaussian_netcdf" # For FV3GFS data on a Gaussian grid in netcdf format. tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - fn_atm="${EXTRN_MDL_FNS[0]}" fn_sfc="${EXTRN_MDL_FNS[1]}" - input_type="gaussian_netcdf" # For FV3-GFS Gaussian grid in netcdf format. convert_nst=True - fi vgtyp_from_climo=True sotyp_from_climo=True diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index ef1c683f36..1ec989c88d 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -324,13 +324,18 @@ case "${EXTRN_MDL_NAME_LBCS}" in "FV3GFS") if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then external_model="FV3GFS" - input_type="gaussian_nemsio" # For FV3-GFS Gaussian grid in nemsio format. + input_type="gaussian_nemsio" # For FV3GFS data on a Gaussian grid in nemsio format. tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then external_model="GFS" fn_grib2="${EXTRN_MDL_FNS[0]}" input_type="grib2" + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "netcdf" ]; then + external_model="FV3GFS" + input_type="gaussian_netcdf" # For FV3GFS data on a Gaussian grid in netcdf format. + tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" + tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" fi ;; @@ -403,6 +408,8 @@ for (( i=0; i<${num_fhrs}; i++ )); do fn_atm="${EXTRN_MDL_FNS[$i]}" elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then fn_grib2="${EXTRN_MDL_FNS[$i]}" + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "netcdf" ]; then + fn_atm="${EXTRN_MDL_FNS[$i]}" fi ;; "RAP") diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh new file mode 100644 index 0000000000..929b91d199 --- /dev/null +++ b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh @@ -0,0 +1,19 @@ +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +FCST_LEN_HRS="06" +LBC_SPEC_INTVL_HRS="3" + +DATE_FIRST_CYCL="20210620" +DATE_LAST_CYCL="20210620" +CYCL_HRS=( "00" ) + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="netcdf" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="netcdf" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt index 5229c95a90..34126824b1 100644 --- a/tests/baselines_list.txt +++ b/tests/baselines_list.txt @@ -13,6 +13,7 @@ get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818 get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518 get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600 get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100 +get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000 get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS get_from_HPSS_ics_HRRR_lbcs_RAP get_from_HPSS_ics_RAP_lbcs_RAP diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index e2588a2ebd..f2224559ae 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -377,42 +377,23 @@ fi fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - -# -#This whole section needs to be set based on what is in HPSS. They are placeholders for now. -# - fns=( "" "" ) - suffix="" - fns=( "" ) - + fns=( "atm" "sfc" ) + suffix="anl.nc" + fns=( "${fns[@]/%/$suffix}" ) + # Set names of external files if searching on disk. if [ "${MACHINE}" = "JET" ]; then - prefix="" + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." else - prefix="" + prefix="gfs.t${hh}z." fi - fns_on_disk=( "" ) + fns_on_disk=( "${fns[@]/#/$prefix}" ) # Set names of external files if searching in an archive file, e.g. from # HPSS. - prefix="" - fns_in_arcv=( "" ) - - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}")")" - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}")")" - - print_info_msg " -Fetching of external model files from NOAA HPSS is not yet supported for -this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) -combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" -Setting fns_on_disk and fns_in_arcv to arrays containing empty elements: - fns_on_disk = ${fns_on_disk_str} - fns_in_arcv = ${fns_in_arcv_str} -If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the -workflow to look for the external model files in a user-staged directory." + prefix="gfs.t${hh}z." + fns_in_arcv=( "${fns[@]/#/$prefix}" ) fi ;; @@ -486,6 +467,7 @@ and analysis or forecast (anl_or_fcst): ;; "FV3GFS") + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) @@ -509,37 +491,21 @@ and analysis or forecast (anl_or_fcst): fns_on_disk=( "${fcst_hhh[@]/#/$prefix}" ) fns_in_arcv=( "${fcst_hhh[@]/#/$prefix}" ) - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - fcst_hhh=( "" ) - suffix="" - fns=( "" ) + fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) + suffix=".nc" + fns=( "${fcst_hhh[@]/%/$suffix}" ) if [ "${MACHINE}" = "JET" ]; then - prefix="" + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" else - prefix="" + prefix="gfs.t${hh}z.atmf" fi - fns_on_disk=( "" ) - - prefix="" - fns_in_arcv=( "" ) - - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}")")" - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}")")" + fns_on_disk=( "${fns[@]/#/$prefix}" ) - print_info_msg " -Fetching of external model files from NOAA HPSS is not yet supported for -this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) -combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" -Setting fns_on_disk and fns_in_arcv to arrays containing empty elements: - fns_on_disk = ${fns_on_disk_str} - fns_in_arcv = ${fns_in_arcv_str} -If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the -workflow to look for the external model files in a user-staged directory." + prefix="gfs.t${hh}z.atmf" + fns_in_arcv=( "${fns[@]/#/$prefix}" ) fi ;; @@ -826,7 +792,7 @@ has not been specified for this external model: # 1) The type of the archive file (e.g. tar, zip, etc). # 2) The name of the archive file. # 3) The full path in HPSS to the archive file. -# 4) The relative directory in the archive file in which the module output +# 4) The relative directory in the archive file in which the model output # files are located. # # Note that these will be used by the calling script only if the archive @@ -848,7 +814,7 @@ has not been specified for this external model: arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" fi arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="$arcv_dir/$arcv_fns" + arcv_fps="${arcv_dir}/${arcv_fns}" ;; "FV3GFS") @@ -889,33 +855,20 @@ has not been specified for this external model: elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="" + arcv_fns="${arcv_fns}gfs_nca" elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_netcdfa="" - first_lbc_fhr="" - last_lbc_fhr="" + last_fhr_in_netcdfa="39" + first_lbc_fhr="${lbc_spec_fhrs[0]}" + last_lbc_fhr="${lbc_spec_fhrs[-1]}" if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then - arcv_fns="" + arcv_fns="${arcv_fns}gfs_nca" elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then - arcv_fns="" + arcv_fns="${arcv_fns}gfs_ncb" else - arcv_fns=( "${arcv_fns}" "${arcv_fns}" ) + arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) fi fi - arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}")")" - - print_info_msg " -Fetching of external model files from NOAA HPSS is not yet supported for -this external model (extrn_mdl_name) and file format (fv3gfs_file_fmt) -combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" -Setting arcv_fns to an array containing empty elements: - arcv_fns = ${arcv_fns_str} -If USE_USER_STAGED_EXTRN_FILES is set to \"TRUE\", this will allow the -workflow to look for the external model files in a user-staged directory." - fi arcv_fmt="tar" @@ -930,11 +883,11 @@ workflow to look for the external model files in a user-staged directory." if [ "$?" = "0" ]; then suffix=".${arcv_fmt}" arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) - prefix="$arcv_dir/" + prefix="${arcv_dir}/" arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) else arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="$arcv_dir/$arcv_fns" + arcv_fps="${arcv_dir}/${arcv_fns}" fi ;; @@ -969,7 +922,7 @@ workflow to look for the external model files in a user-staged directory." arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" arcv_fmt="zip" arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="$arcv_dir/$arcv_fns" + arcv_fps="${arcv_dir}/${arcv_fns}" arcvrel_dir="" # # Reset hh to its original value in case it is used again later below. @@ -985,7 +938,7 @@ workflow to look for the external model files in a user-staged directory." arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" arcv_fmt="zip" arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="$arcv_dir/$arcv_fns" + arcv_fps="${arcv_dir}/${arcv_fns}" arcvrel_dir="" ;; @@ -993,7 +946,7 @@ workflow to look for the external model files in a user-staged directory." arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" arcv_fmt="tar" arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.bgrid.${arcv_fmt}" - arcv_fps="$arcv_dir/$arcv_fns" + arcv_fps="${arcv_dir}/${arcv_fns}" arcvrel_dir="" ;; From d8b401b4b0cadefd87e6cbcf50f67e631bfb0b16 Mon Sep 17 00:00:00 2001 From: JulieSchramm Date: Fri, 23 Jul 2021 12:09:19 -0600 Subject: [PATCH 069/203] Add COMPILER="intel" to config.community.sh and config.nco.sh --- ush/config.community.sh | 1 + ush/config.nco.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ush/config.community.sh b/ush/config.community.sh index 8bbe0587ed..b0c1f21f1b 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -2,6 +2,7 @@ MACHINE="hera" ACCOUNT="an_account" EXPT_SUBDIR="test_community" +COMPILER="intel" VERBOSE="TRUE" RUN_ENVIR="community" diff --git a/ush/config.nco.sh b/ush/config.nco.sh index 58a4c01003..3b2c84be68 100644 --- a/ush/config.nco.sh +++ b/ush/config.nco.sh @@ -2,6 +2,7 @@ MACHINE="hera" ACCOUNT="an_account" EXPT_SUBDIR="test_nco" +COMPILER="intel" VERBOSE="TRUE" RUN_ENVIR="nco" From c2ce7c5b1bff54563b8f51945e2fc82711dc070d Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:13:26 -0600 Subject: [PATCH 070/203] Reorganize the WE2E (workflow end-to-end) testing system (#531) ## DESCRIPTION OF CHANGES: This PR reorganizes the WE2E testing system so that it is easier to comprehend, use, and modify. The major changes are as follows: * Move the WE2E testing system from the `regional_workflow/tests` directory to `regional_workflow/tests/WE2E`. This is because we anticipate other types of test in the testing system, e.g. unit tests, which would go under `regional_workflow/tests/unit`, etc. * Move the WE2E test configuration files that were in `regional_workflow/tests/baseline_configs` to category subdirectories under `regional_workflow/tests/WE2E/test_configs`. The category subdirectories thus far and the types of tests they contain are: * `grids_extrn_mdls_suites_community` Tests in community mode of various combinations of the grid, external models for ICs and LBCs, and physics suites. * `grids_extrn_mdls_suites_nco` Tests in NCO mode of various combinations of the grid, external models for ICs and LBCs, and physics suites. * `release_SRW_v1` The graduate student test (GST) used for the UFS SRW App version 1 release. * `wflow_features` Test of workflow features, e.g. ability to set various parameters to user-specified values instead of using the defaults, ability to fetch external model files from different models and on different dates from NOAA-HPSS, etc. * Rename some of the WE2E test configuration files to adhere to the naming convention used in each category subdirectory. * Remove some of the WE2E test configuration files since they are almost-duplicates, e.g. they differ with respect to another test only in the cycle date/time used or the LBC specification interval. * Changes to contents of test configuration files: * Rearrange the order in which experiment variables are specified in the test configuration files so that the predefined grid name and the physics suite are set first, then the external model info is set, then the cycle dates, then the forecast length and LBC update interval, finally followed by other parameters. * Remove redundant variable specifications in the test configuration files, e.g. remove `QUILTING="TRUE"` since `QUILTING` is already set to `"TRUE"` by default, remove `GRID_GEN_METHOD="ESGgrid"` since a predefined grid is already specified that in turn implies a value for `GRID_GEN_METHOD`. * Include a test purpose/description in each test configuration file. * Perform bug fixes in the WE2E configuration files, e.g. a test that was supposed to run on the `RRFS_CONUS_3km` grid was actually running on the `RRFS_CONUS_13km` grid. * Remove the file `baselines_list.txt` since it it not used. * Add new function in `get_WE2Etest_names_subdirs_descs.sh` that: * Searches subdirectories under the base directory in which the WE2E test configuration files are located (`regional_workflow/tests/WE2E/test_configs`) and returns a list of all available test names, the category subdirectories under the base directory in which they are located, the unique test IDs, and the test descriptions. * Creates (if requested) a comma-separated value (CSV) file containing this WE2E information that can be opened as a spreadsheet in Google Sheets. * Rename `run_experiments.sh` to `run_WE2E_tests.sh`. * In `run_WE2E_tests.sh`: * Include a detailed usage message. * Make sure that required arguments are provided on the command line. * Call the new function `get_WE2Etest_names_subdirs_descs` to get a full list of all available tests. Then check the list of test names that the user wants to run to make sure all exist in the full list. * Run sanity checks on the user-specified list of tests to run, e.g. that a test is not repeated (either under the same name or under an alternate name). * In `ush/bash_utils/filesys_cmds_vrfy.sh`, put the `local` attribute in front of variables that are supposed to be local. * In `ush/bash_utils/is_element_of.sh`, add the `:-` at the end of `array_name_at` so that the function still works when the array passed in is empty. * In `ush/config_defaults.sh`, edit comments and move groups of variables to more appropriate location in file. ## TESTS CONDUCTED: On Hera, ran the following tests so far: * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000 * inline_post * specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS * specify_RESTART_INTERVAL All were successful. Since it is expensive to run all the new WE2E tests, the remainder will be tested gradually and the results recorded in a google spreadsheet (for now; this will have to be automated at some point). ## DEPENDENCIES: There will likely be a companion PR into `ufs-weather-app` that updates the hash of `regional_workflow`, but that can only be created after this PR is merged. ## DOCUMENTATION: Detailed documentation for how the new WE2E testing system works is in the comments in the files `run_WE2E_tests.sh` and `get_WE2Etest_names_subdirs_descs.sh`. Transferring these to RST files will take time and is likely better to do as part of a separate PR. --- jobs/JREGIONAL_RUN_POST | 4 +- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 1270 +++++++++++++++++ tests/{ => WE2E}/get_expts_status.sh | 0 tests/WE2E/run_WE2E_tests.sh | 1214 ++++++++++++++++ ...id_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 26 + ...id_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 26 + ...AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} | 26 +- ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} | 21 +- ...AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} | 21 +- ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} | 21 +- ..._AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} | 21 +- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 22 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 22 +- ...US_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 23 +- ...CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 23 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 24 +- ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 23 +- ...3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh} | 21 +- ..._FV3GFS_suite_GFS_2017_gfdlmp_regional.sh} | 22 +- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 22 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 22 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh | 22 +- ...NUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh | 31 + ...km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh} | 21 +- ...MGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh} | 23 +- ...ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh} | 21 +- ...m_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh} | 21 +- ...S_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh | 22 +- ...ONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh | 22 +- ...km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh | 22 +- ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 23 +- ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh} | 22 +- ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 23 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 24 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} | 29 +- ...NUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh | 22 +- ..._CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh | 22 +- ...25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh | 22 +- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 22 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 22 +- ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 26 + ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 23 +- ..._CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 23 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 24 +- ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 23 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh} | 21 +- ..._FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh} | 25 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 22 +- ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 23 +- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 23 +- ...id_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 26 + ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 21 +- ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 21 +- ...s_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh | 21 +- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 25 +- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 21 +- ..._CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 21 +- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 21 +- .../config.GST_release_public_v1.sh | 26 +- .../config.MET_verification.sh | 19 +- .../config.community_ensemble_008mems.sh | 31 + .../config.community_ensemble_2mems.sh | 35 + ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh | 27 + ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh | 29 + ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh | 28 + ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh | 28 + ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh | 28 + ...HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh} | 19 +- ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh | 27 + ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh | 29 + ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh | 28 + ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh | 28 + ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh | 28 + ...V3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh | 23 +- ...ig.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh | 19 +- .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 26 + .../config.get_from_HPSS_ics_RAP_lbcs_RAP.sh | 19 +- ...MADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh} | 19 +- .../wflow_features}/config.inline_post.sh | 6 +- .../wflow_features/config.nco_ensemble.sh | 35 + .../wflow_features}/config.nco_inline_post.sh | 0 .../wflow_features/config.new_ESGgrid.sh | 62 + .../wflow_features}/config.new_GFDLgrid.sh | 55 +- ..._USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh | 44 +- ...d_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh | 44 +- .../config.pregen_grid_orog_sfc_climo.sh | 20 +- .../config.specify_DOT_OR_USCORE.sh | 39 + ...g.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh} | 31 +- ...g.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh | 17 +- .../config.specify_RESTART_INTERVAL.sh} | 25 +- .../wflow_features}/config.subhourly_post.sh | 21 +- .../config.subhourly_post_ensemble_2mems.sh | 38 + .../config.community_ensemble_008mems.sh | 21 - .../config.community_ensemble_2mems.sh | 21 - ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh | 19 - ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh | 19 - ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh | 19 - ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh | 19 - ...FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh | 19 - ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh | 19 - ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh | 19 - ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh | 19 - ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh | 19 - ...V3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh | 19 - .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 17 - ....grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS.sh | 17 - ...g.grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS.sh | 17 - ...NUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh | 19 - ...5km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 17 - ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 17 - tests/baseline_configs/config.nco_ensemble.sh | 21 - ...id_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 18 - tests/baseline_configs/config.new_ESGgrid.sh | 49 - .../config.suite_FV3_GFS_2017_gfdlmp.sh | 18 - .../config.suite_FV3_GSD_SAR.sh | 18 - .../baseline_configs/config.suite_FV3_HRRR.sh | 17 - .../config.suite_FV3_RRFS_v1alpha.sh | 19 - .../config.suite_FV3_RRFS_v1beta.sh | 18 - tests/baselines_list.txt | 86 -- tests/run_experiments.sh | 994 ------------- tests/testlist.release_public_v1.txt | 7 - ush/bash_utils/filesys_cmds_vrfy.sh | 10 +- ush/bash_utils/is_element_of.sh | 2 +- ush/config_defaults.sh | 97 +- 124 files changed, 4231 insertions(+), 2152 deletions(-) create mode 100755 tests/WE2E/get_WE2Etest_names_subdirs_descs.sh rename tests/{ => WE2E}/get_expts_status.sh (100%) create mode 100755 tests/WE2E/run_WE2E_tests.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename tests/{baseline_configs/config.grid_GSD_HRRR_AK_50km.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} (63%) rename tests/{baseline_configs/config.grid_RRFS_AK_13km_FV3GFS_FV3GFS.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} (52%) rename tests/{baseline_configs/config.grid_RRFS_AK_13km_RAP_RAP.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} (52%) rename tests/{baseline_configs/config.grid_RRFS_AK_3km_FV3GFS_FV3GFS.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} (52%) rename tests/{baseline_configs/config.grid_RRFS_AK_3km_RAP_RAP.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh} (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh (53%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh (50%) rename tests/{baseline_configs/config.regional_002.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh} (52%) rename tests/{baseline_configs/config.suite_FV3_GFS_2017_gfdlmp_regional.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh} (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh (52%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh rename tests/{baseline_configs/config.suite_FV3_CPT_v0.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh} (52%) rename tests/{baseline_configs/config.DOT_OR_USCORE.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh} (52%) rename tests/{baseline_configs/config.suite_FV3_GFS_v15p2.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh} (52%) rename tests/{baseline_configs/config.suite_FV3_GFS_v16.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh} (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (53%) rename tests/{baseline_configs/config.suite_FV3_GSD_v0.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh} (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh (53%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh (53%) rename tests/{baseline_configs/config.subhourly_post_ensemble_2mems.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} (53%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh (55%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh (52%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh (53%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh (50%) rename tests/{baseline_configs/config.grid_RRFS_NA_13km.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh} (52%) rename tests/{baseline_configs/config.grid_RRFS_NA_3km.sh => WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh} (71%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_community}/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh (52%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh (51%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/grids_extrn_mdls_suites_nco}/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh (50%) rename tests/{baseline_configs => WE2E/test_configs/release_SRW_v1}/config.GST_release_public_v1.sh (66%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.MET_verification.sh (96%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh rename tests/{baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh => WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh} (50%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh (59%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh (53%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh (52%) rename tests/{baseline_configs/config.user_download_extrn_files.sh => WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh} (54%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.inline_post.sh (99%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.nco_inline_post.sh (100%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.new_GFDLgrid.sh (63%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh (77%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh (76%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.pregen_grid_orog_sfc_climo.sh (66%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh rename tests/{baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh => WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh} (55%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh (64%) rename tests/{baseline_configs/config.restart_interval.sh => WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh} (54%) rename tests/{baseline_configs => WE2E/test_configs/wflow_features}/config.subhourly_post.sh (65%) create mode 100644 tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh delete mode 100644 tests/baseline_configs/config.community_ensemble_008mems.sh delete mode 100644 tests/baseline_configs/config.community_ensemble_2mems.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh delete mode 100644 tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh delete mode 100644 tests/baseline_configs/config.grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS.sh delete mode 100644 tests/baseline_configs/config.grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS.sh delete mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh delete mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh delete mode 100644 tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh delete mode 100644 tests/baseline_configs/config.nco_ensemble.sh delete mode 100644 tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh delete mode 100644 tests/baseline_configs/config.new_ESGgrid.sh delete mode 100644 tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp.sh delete mode 100644 tests/baseline_configs/config.suite_FV3_GSD_SAR.sh delete mode 100644 tests/baseline_configs/config.suite_FV3_HRRR.sh delete mode 100644 tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh delete mode 100644 tests/baseline_configs/config.suite_FV3_RRFS_v1beta.sh delete mode 100644 tests/baselines_list.txt delete mode 100755 tests/run_experiments.sh delete mode 100644 tests/testlist.release_public_v1.txt diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index fca5ae21a4..68dcf95bd4 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -98,9 +98,9 @@ fi mkdir_vrfy -p "${postprd_dir}" if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then -tmp_dir="${postprd_dir}/$fhr$fmn" + tmp_dir="${postprd_dir}/$fhr$fmn" else -tmp_dir="${postprd_dir}/$fhr" + tmp_dir="${postprd_dir}/$fhr" fi check_for_preexist_dir_file "${tmp_dir}" "delete" mkdir_vrfy -p "${tmp_dir}" diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh new file mode 100755 index 0000000000..257cd89129 --- /dev/null +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -0,0 +1,1270 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This file defines a function that gathers and returns information about +# the WE2E tests available in the WE2E testing system. This information +# consists of the test names, the category subdirectories in which the +# test configuration files are located (relative to a base directory), +# the test IDs, and the test descriptions. These are described in more +# detail below. +# +# The function takes as inputs the following arguments: +# +# WE2Edir: +# Directory in which the WE2E testing system is located. This system +# consists of the main script for running WE2E tests, various auxiliary +# scripts, and the test configuration files. +# +# generate_csv_file: +# Flag that specifies whether or not a CSV (Comma-Separated Value) file +# containing information about the WE2E tests should be generated. +# +# output_varname_test_configs_basedir: +# Name of output variable in which to return the base directory of the +# WE2E test configuration files. +# +# output_varname_test_names: +# Name of output array variable in which to return the names of the WE2E +# tests. +# +# output_varname_test_subdirs: +# Name of output array variable in which to return the category subdirectories +# in which the WE2E tests are located. +# +# output_varname_test_ids: +# Name of output array variable in which to return the IDs of the WE2E +# tests. +# +# output_varname_test_descs: +# Name of output array variable in which to return the descriptions of +# the WE2E tests. +# +# Note that any input argument that is not specified in the call to this +# function gets set to a null string in the body of the function. In +# particular, if any of the arguments that start with "output_varname_" +# (indicating that they specify the name of an output variable) are not +# set in the call, the values corresponding to those variables are not +# returned to the calling script or function. +# +# In order to gather information about the available WE2E tests, this +# function sets the local variable test_configs_basedir to the full path +# of the base directory in which the test configuration files (which may +# be ordinary files or symlinks) are located. It sets this as follows: +# +# test_configs_basedir="${WE2Edir}/test_configs" +# +# If the argument output_varname_test_configs_basedir is specified in +# the call to this function, then the value of test_configs_basedir will +# be returned to the calling script or function (in the variable specified +# by output_varname_test_configs_basedir). +# +# The WE2E test configuration files are located in subdirectories under +# the base directory. This function sets the names of these subdirectories +# in the local array category_subdirs. We refer to these as "category" +# subdirectories because they are used for clarity to group the tests +# into categories (instead of putting them all directly under the base +# directory). For example, one category of tests might be those that +# test workflow capabilities such as running multiple cycles and ensemble +# forecasts, another might be those that run various combinations of +# grids, physics suites, and external models for ICs/LBCs, etc. Note +# that if a new category subdirectory is added under test_configs_basedir, +# its name must be added below as a new element in category_subdirs; +# otherwise, this new subdirectory will not be searched for test +# configuration files. Note also that if one of the elements of +# category_subdirs is ".", then this function will also search directly +# under the base directory itself for test configuration files. +# +# Once test_configs_basedir and category_subdirs are set, this function +# searches the category subdirectories for WE2E test configuration files. +# In doing so, it assumes that any ordinary file or symlink in the category +# subdirectories having a name of the form +# +# config.${test_name}.sh +# +# is a test configuration file, and it takes the name of the corresponding +# test to be given by whatever test_name in the above file name happens +# to be. Here, by "ordinary" file we mean an item in the file system +# that is not a symlink (or a directory or other more exotic entity). +# Also, for simplicity, we require that any configuration file that is a +# symlink have a target that is an ordinary configuration file, i.e. not +# a symlink. +# +# We allow test configuration files to be symlinks in order to avoid the +# presence of identical configuration files with different names in the +# WE2E testing system. For example, assume there is a test named +# "test_grid1" that is used to test whether the forecast model can run +# on a grid named "grid1", and assume that the configuration file for +# this test is an ordinary file located in a category subdirectory named +# "grids" that contains tests for various grids. Then the full path to +# this configuration file will be +# +# ${test_configs_basedir}/grids/config.test_grid1.sh +# +# Now assume that there is another category subdirectory named "suites" +# that contains configuration files for tests that check whether the +# forecast model can run with various physics suites. Thus, in order to +# have a test that checks whether the forecast model can run successfully +# with a physics suite named "suite1", we might create an ordinary +# configuration file named "config.test_suite1.sh" in "suites" (so that +# the corresponding test name is "test_suite1"). Thus, the full path to +# this configuration file would be +# +# ${test_configs_basedir}/suites/config.test_suite1.sh +# +# Now if test "test_grid1" happens to use physics suite "suite1", then +# we may be able to use that test for testing both "grid1" and "suite1". +# However, we'd still want to have a configuration file in the "suites" +# subdirectory with a test name that makes it clear that the purpose of +# the test is to run using "suite1". Then, since the WE2E testing system +# allows configuration files to by symlinks, instead of copying +# "config.test_grid1.sh" from the "grids" to the "suites" subdirectory +# and renaming it to "config.test_suite1.sh" (which would create two +# identical ordinary configuration files), we could simply make +# "config.test_suite1.sh" in "suites" a symlink to "config.test_grid1.sh" +# in "grids", i.e. +# +# ${test_configs_basedir}/suites/config.test_suite1.sh +# --> ${test_configs_basedir}/grids/config.test_grid1.sh +# +# With this approach, there will be only one ordinary configuration file +# to maintain. Note that there may be more than one symlink pointing to +# the same ordinary configuration file. For example, there may be another +# category subdirectory named "wflow_features" containing tests for +# various workflow features. Then if the test "test_grid1" runs a test +# that, in addition to running the forecast model on "grid1" using the +# "suite1" physics suite also performs subhourly output, then a symlink +# named "config.test_subhourly.sh" can be created under "wflow_features" +# that points to the configuration file "config.test_grid1.sh", i.e. +# +# ${test_configs_basedir}/wflow_features/config.test_subhourly.sh +# --> ${test_configs_basedir}/grids/config.test_grid1.sh +# +# Since the WE2E testing system allows configuration files to be symlinks, +# the same WE2E test may be referred to via multiple test names -- the +# test name corresponding to the ordinary configuration file ("test_grid1" +# in the example above) and any one of the test names corresponding to +# any symlinks that have this ordinary file as their target ("test_suite1" +# and "test_subhourly" in the example above). Here, for clarity we will +# refer to the test name derived from the name of the ordinary configuration +# file as the "primary" test name, and we will refer to the test names +# dervied from the symlinks as the alternate test names. Since these +# test names all represent the same actual test, we also assign to each +# group of primary and alternate test names a single test ID. This is +# simply an integer that uniquely identifies each group of primary and +# alternate test names. +# +# For each configuration file (which may be an ordinary file or a symlink) +# found in the category subdirectories, this function saves in local +# arrays the following information about the WE2E files: +# +# 1) The list of all available WE2E test names, both primary and alternate. +# 2) The category subdirectories under the base directory test_configs_basedir +# in which the test configuration files corresponding to each test +# name are located. +# 3) The IDs corresponding to each of the test names. +# 4) The test descriptions (if output_varname_test_descs is specified in +# the call to this function or if generate_csv_file is or gets set to +# "TRUE"; see below). +# +# These local arrays are sorted in order of increasing test ID. Within +# each group of tests that have the same ID, the primary test name is +# listed first followed by zero or more alternate test names. Note also +# that to reduce confusion, we do not allow two or more configuration +# files of the same name anywere under test_configs_basedir (either +# representing the same actual test or different ones). In other words, +# the list of all test names that this function generates cannot contain +# any duplicate names (either primary or alternate). After assembling +# the full list of test names, this function checks for such duplicates +# and exits with an error message if any are found. +# +# The following input arguments to this function specify the names of +# the arrays in which each of the quantities listed above should be +# returned (to the calling script or function): +# +# output_varname_test_names +# output_varname_test_subdirs +# output_varname_test_ids +# output_varname_test_descs +# +# If any of these is not specified in the call to this function, then +# the corresponding quantity will not be returned to the calling script +# or function. +# +# The test descriptions are headers consisting of one or more bash-style +# comment lines at the top of each ordinary test configuraiton file. +# They are extracted from each such file and placed in a local array only +# if one or both of the following conditions are met: +# +# 1) The user explicitly asks for the descriptions to be returned by +# specifying in the call to this function the name of the array in +# which to return them (by setting a value for the argument +# output_varname_test_descs). +# 2) A CSV file summarizing the WE2E tests will be generated (see below) +# +# For convenience, this function can generate a CSV (comma-separated +# value) file containing information about the WE2E tests. If it does, +# the file will be placed in the main WE2E testing system directory +# specified by the input argument WE2Edir. The CSV file can be read +# into a spreadsheet in Google Sheets (or another similar tool) to get +# an overview of all the available WE2E tests. +# +# A CSV file will be generated in the directory specified by WE2Edir if +# one or more of the following conditions hold: +# +# 1) The input argument generate_csv_file is set to "TRUE" in the call +# to this function. +# 2) The input argument generate_csv_file is not set in the call to this +# function, and a CSV file does not already exist. +# 3) The input argument generate_csv_file is not set in the call to this +# function, a CSV file already exists, and the modification time of +# at least one category subdirectory in category_subdirs is later +# than that of the CSV file, i.e. the existing CSV file needs to be +# updated because the test configuration files may have changed in +# some way. +# +# A CSV file is not generated if generate_csv_file is explicitly set to +# "FALSE" in the call to this function (regardless of whether or not a +# CSV file already exists). If a CSV file is generated, it is placed in +# the directory specified by the input argment WE2Edir, and it overwrites +# any existing copies of the file in that directory. +# +#----------------------------------------------------------------------- +# +function get_WE2Etest_names_subdirs_descs() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script or function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script or function. +# Then process the arguments provided to it on the command line (which +# should consist of a set of name-value pairs of the form arg1="value1", +# arg2="value2", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "WE2Edir" \ + "generate_csv_file" \ + "output_varname_test_configs_basedir" \ + "output_varname_test_names" \ + "output_varname_test_subdirs" \ + "output_varname_test_ids" \ + "output_varname_test_descs" \ + ) + process_args "valid_args" "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local all_items \ + alt_test_name \ + alt_test_names \ + alt_test_names_subdirs \ + alt_test_prim_test_names \ + alt_test_subdir \ + alt_test_subdirs \ + category_subdirs \ + column_titles \ + config_fn \ + crnt_item \ + csv_delimiter \ + csv_fn \ + csv_fp \ + cwd \ + hash_or_null \ + i \ + ii \ + j \ + jp1 \ + line \ + mod_time_csv \ + mod_time_subdir \ + num_alt_tests \ + num_category_subdirs \ + num_items \ + num_occurrences \ + num_prim_tests \ + num_tests \ + prim_test_descs \ + prim_test_ids \ + prim_test_name_subdir \ + prim_test_names \ + prim_test_subdirs \ + get_test_descs \ + regex_search \ + row_content \ + sort_inds \ + stripped_line \ + subdir \ + subdir_fp \ + subdirs \ + target_dir \ + target_fn \ + target_fp \ + target_prim_test_name \ + target_rp \ + target_test_name_or_null \ + test_configs_basedir \ + test_desc \ + test_descs \ + test_descs_esc_sq \ + test_descs_orig \ + test_descs_str \ + test_id \ + test_id_next \ + test_ids \ + test_ids_and_inds \ + test_ids_and_inds_sorted \ + test_ids_orig \ + test_ids_str \ + test_name \ + test_name_or_null \ + test_names \ + test_names_orig \ + test_names_str \ + test_subdirs \ + test_subdirs_orig \ + test_subdirs_str \ + test_type \ + valid_vals_generate_csv_file +# +#----------------------------------------------------------------------- +# +# Set variables associated with the CSV (comma-separated value) file that +# this function may generate. The conditions under which such a file is +# generated are described above in the description of this function. +# +#----------------------------------------------------------------------- +# +# Set the name and full path to the CSV file. +# + csv_fn="WE2E_test_info.csv" + csv_fp="${WE2Edir}/${csv_fn}" +# +# If generate_csv_file is specified as an input argument in the call to +# this function, make sure that it is set to a valid value. +# + if [ ! -z "${generate_csv_file}" ]; then + + valid_vals_generate_csv_file=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") + check_var_valid_value "generate_csv_file" "valid_vals_generate_csv_file" + + generate_csv_file=${generate_csv_file^^} + if [ "${generate_csv_file}" = "TRUE" ] || \ + [ "${generate_csv_file}" = "YES" ]; then + generate_csv_file="TRUE" + elif [ "${generate_csv_file}" = "FALSE" ] || \ + [ "${generate_csv_file}" = "NO" ]; then + generate_csv_file="FALSE" + fi +# +# If generate_csv_file was not specified as an input argument in the +# call to this function, then it will have been set above to a null +# string. In this case, if a CSV file doesn't already exsit, reset +# generate_csv_file to "TRUE" so that one will be generated. If a CSV +# file does exist, get its modification time so that later below, we can +# compare it to the modification times of the category subdirectories +# and determine whether a new CSV file needs to be generated. +# +# Note that the modification "times" obtained here and later below using +# the "stat" utility are the seconds elapsed between Epoch (which is a +# fixed point in time) and the last modification time of the specified +# file, not the dates/times at which the file was last modified. This +# is due to the use of the "--format=%Y" flag in the call to "stat". We +# choose these "seconds since Epoch" units because they make it easier +# to determine which of two files is younger/older (the one with the +# larger seconds-since-Epoch will be the more recently modified file.) +# + else + + if [ ! -f "${csv_fp}" ]; then + mod_time_csv="0" + generate_csv_file="TRUE" + else + mod_time_csv=$( stat --format=%Y "${csv_fp}" ) + fi + + fi +# +#----------------------------------------------------------------------- +# +# Set the base directory containing the WE2E test configuration files +# (or, more precisely, containing the category subdirectories in which +# the configuration files are located). +# +#----------------------------------------------------------------------- +# + test_configs_basedir="${WE2Edir}/test_configs" +# +#----------------------------------------------------------------------- +# +# Set the array category_subdirs that specifies the subdirectories under +# test_configs_basedir in which to search for WE2E test configuration +# files. Note that if "." is included as one of the elements of this +# array, then the base directory itself will also be searched. +# +#----------------------------------------------------------------------- +# + category_subdirs=( \ + "." \ + "grids_extrn_mdls_suites_community" \ + "grids_extrn_mdls_suites_nco" \ + "release_SRW_v1" \ + "wflow_features" \ + ) + num_category_subdirs="${#category_subdirs[@]}" +# +#----------------------------------------------------------------------- +# +# Loop over the category subdirectories under test_configs_basedir +# (possibly including the base directory itself). In each subdirectory, +# consider all items that have names of the form +# +# config.${test_name}.sh +# +# and that are either ordinary files (i.e. not symlinks) or are symlinks +# whose targets are ordinary files having names of the form above. For +# each item that is an ordinary file, save the corresponding primary test +# name, the category subdirectory in which the item is located, and the +# test ID in the arrays +# +# prim_test_names +# prim_test_subdirs +# prim_test_ids +# +# respectively. For each item that is a symlink to an ordinary file, +# save the alternate test name corresponding to the symlink name, the +# category subdirectory in which the symlink is located, and the test +# name derived from the name of the symlink's target (i.e. the primary +# test name that this alternate test name corresponds to) in the arrays +# +# alt_test_names +# alt_test_subdirs +# alt_test_prim_test_names +# +# respectively. +# +#----------------------------------------------------------------------- +# + prim_test_names=() + prim_test_ids=() + prim_test_subdirs=() + + alt_test_names=() + alt_test_subdirs=() + alt_test_prim_test_names=() +# +# Initialize the counter that will be used to assign test IDs to the +# primary test names. This will be incremented below every time a new +# primary test name is found. Note that we do not yet assign IDs to the +# alternate test names. These will be assigned IDs later below that +# will be identical to the IDs of the primary thest names they correspond +# to. +# + test_id="0" + + for (( i=0; i<=$((num_category_subdirs-1)); i++ )); do + + subdir="${category_subdirs[$i]}" + subdir_fp="${test_configs_basedir}/$subdir" +# +# If at this point in the code generate_csv_file is still set to a null +# string, it means that a CSV file containing information about the WE2E +# tests already exists. In this case, a new version of this file needs +# to be generated only if one or more of the category subdirectories +# have modification times that are later than that of the existing CSV +# file. Check for this condition and set generate_csv_file accordingly. +# Note that this if-statement will be executed at most once since it sets +# generate_csv_file to "TRUE", after which the test for entering the if- +# statement will be false. +# + if [ -z "${generate_csv_file}" ]; then + mod_time_subdir=$( stat --format=%Y "${subdir_fp}" ) + if [ "${mod_time_subdir}" -gt "${mod_time_csv}" ]; then + generate_csv_file="TRUE" + print_info_msg " +The current category subdirectory (subdir) has a modification time +(mod_time_subdir) that is later than the modification time (mod_time_csv) +of the existing CSV file (csv_fp) containing WE2E test information: + subdir = \"${subdir}\" + mod_time_subdir = \"${mod_time_subdir}\" (in units of seconds since Epoch) + mod_time_csv = \"${mod_time_csv}\" (in units of seconds since Epoch) + csv_fp = \"${csv_fp}\" +Thus, the CSV file must be updated. Setting generate_csv_file to \"TRUE\" +to generate a new CSV file: + generate_csv_file = \"${generate_csv_file}\"" + fi + fi +# +# Change location to the current category subdirectory. +# + cd_vrfy "${subdir_fp}" +# +# Get the contents of the current subdirectory. We consider each item +# that has a name of the form +# +# config.${test_name}.sh +# +# to be a WE2E test configuration file, and we take the name of the test +# to be whatever ${test_name} in the above expression corresponds to. +# We ignore all other items in the subdirectory. +# + all_items=( $(ls -1) ) + num_items="${#all_items[@]}" + for (( j=0; j<=$((num_items-1)); j++ )); do + + crnt_item="${all_items[$j]}" +# +# Try to extract the name of the test from the name of the current item +# and place the result in test_name_or_null. test_name_or_null will +# contain the name of the test only if the item has a name of the form +# "config.${test_name}.sh", in which case it will be equal to ${test_name}. +# Otherwise, it will be a null string. +# + regex_search="^config\.(.*)\.sh$" + test_name_or_null=$( printf "%s\n" "${crnt_item}" | \ + sed -n -r -e "s/${regex_search}/\1/p" ) +# +#----------------------------------------------------------------------- +# +# Take further action for this item only if it has a name of the form +# above expected for a WE2E test configuration file, which will be the +# case only if test_name_or_null is not a null string. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${test_name_or_null}" ]; then +# +#----------------------------------------------------------------------- +# +# Use bash's -h conditional operator to check whether the current item +# (which at this point is taken to be a test configuration file) is a +# symlink. If it is a symlink, the only type of entity we allow the +# target to be is an existing ordinary file. In particular, to keep the +# WE2E testing system simple, we do not allow the target to be a symlink. +# Of course, it also cannot be a directory or other exotic entity. Below, +# we check for these various possibilities and only allow the case of the +# target being an existing ordinary file. +# +#----------------------------------------------------------------------- +# + if [ -h "${crnt_item}" ]; then +# +# Extract the name of the test from the name of the symlink and append +# it to the array alt_test_names. Also, append the category subdirectory +# under test_configs_basedir in which the symlink is located to the array +# alt_test_subdirs. +# + alt_test_names+=("${test_name_or_null}") + alt_test_subdirs+=("$subdir") +# +# Get the full path to the target of the symlink without following targets +# that are themselves symlinks. The "readlink" utility without any flags +# (such as -f) can do this, but when -f is omitted, it returns a relative +# path. To convert that relative path to an absolute path without resolving +# symlinks, use the "realpath" utility with the -s flag. +# + target_rp=$( readlink "${crnt_item}" ) + target_fp=$( realpath -s "${target_rp}" ) +# +# Use bash's -h conditional operator to check whether the target itself +# is a symlink. For simplicity, this is not allowed. Thus, in this +# case, print out an error message and exit. +# + if [ -h "${target_fp}" ]; then + cwd="$(pwd)" + print_err_msg_exit "\ +The symlink (crnt_item) in the current directory (cwd) has a target +(target_fp) that is itself a symlink: + cwd = \"${cwd}\" + crnt_item = \"${crnt_item}\" + target_fp = \"${target_fp}\" +This is not allowed. Please ensure that the current item points to an +ordinary file (i.e. not a symlink) and rerun." + fi +# +# Now use bash's -f conditional operator to check whether the target is +# a "regular" file (as defined by bash). Note that this test will return +# false if the target is a directory or does not exist and true otherwise. +# Thus, the negation of this test applied to the target (i.e. ! -f) that +# we use below will be true if the target is not an existing file. In +# this case, we print out an error message and exit. +# +# Note also that the -f operator recursively follows a symlink passed to +# it as an argument. For this reason, we need to first perform the -h +# test above to check that the target (without resolving symlinks) is +# itself not a symlink. The -f test below does not help in this regard. +# + if [ ! -f "${target_fp}" ]; then + cwd="$(pwd)" + print_err_msg_exit "\ +The symlink (crnt_item) in the current directory (cwd) has a target +(target_fp) that is not an existing ordinary file: + cwd = \"${cwd}\" + crnt_item = \"${crnt_item}\" + target_fp = \"${target_fp}\" +This is probably because either the target doesn't exist or is a directory, +neither of which is allowed because the symlink must point to an ordinary +(i.e. non-symlink) WE2E test configuration file. Please either point the +symlink to such a file or remove it, then rerun." + fi +# +# Get the name of the directory in which the target is located. +# + target_dir=$( dirname "${target_fp}" ) +# +# Next, check whether the directory in which the target is located is +# under the base directory of the WE2E test configuration files (i.e. +# test_configs_basedir). We require that the target be located in one +# of the subdirectories under test_configs_basedir (or directly under +# test_configs_basedir itself) because we don't want to deal with tests +# that have configuration files that may be located anywhere in the file +# system; for simplicity, we want all configuration files to be placed +# somewhere under test_configs_basedir. +# +# Note that the bash parameter expansion ${var/search/replace} returns +# $var but with the first instance of "search" replaced by "replace" if +# the former is found in $var. Otherwise, it returns the original $var. +# If "replace" is omitted, then "search" is simply deleted. Thus, in +# the if-statement below, if ${target_dir/${test_configs_basedir}/} +# returns ${target_dir} without changes (in which case the test in the +# if-statment will evaluate to true), it means ${test_configs_basedir} +# was not found within ${target_dir}. That in turn means ${target_dir} +# is not a location under ${test_configs_basedir}. In this case, print +# out a warning and exit. +# + if [ "${target_dir}" = "${target_dir/${test_configs_basedir}/}" ]; then + cwd="$(pwd)" + print_err_msg_exit "\ +The symlink (crnt_item) in the current directory (cwd) has a target +(target_fp) located in a directory (target_dir) that is not somewhere +under the WE2E tests base directory (test_configs_basedir): + cwd = \"${cwd}\" + crnt_item = \"${crnt_item}\" + target_fp = \"${target_fp}\" + target_dir = \"${target_dir}\" + test_configs_basedir = \"${test_configs_basedir}\" +For clarity, we require all WE2E test configuration files to be located +somewhere under test_configs_basedir (either directly in this base +directory on in a subdirectory). Please correct and rerun." + fi +# +# Finally, check whether the name of the target file is in the expected +# format "config.${test_name}.sh" for a WE2E test configuration file. +# If not, print out a warning and exit. +# + target_fn=$( basename "${target_fp}" ) + target_test_name_or_null=$( printf "%s\n" "${target_fn}" | \ + sed -n -r -e "s/${regex_search}/\1/p" ) + if [ -z "${target_test_name_or_null}" ]; then + cwd="$(pwd)" + print_err_msg_exit "\ +The symlink (crnt_item) in the current directory (cwd) has a target +(target_fn; located in the directory target_dir) with a name that is +not in the form \"config.[test_name].sh\" expected for a WE2E test +configuration file: + cwd = \"${cwd}\" + crnt_item = \"${crnt_item}\" + target_dir = \"${target_dir}\" + target_fn = \"${target_fn}\" +Please either rename the target to have the form specified above or +remove the symlink, then rerun." + fi +# +# Now that all the checks above have succeeded, for later use save the +# name of the WE2E test that the target represents in the array +# alt_test_prim_test_names. +# + alt_test_prim_test_names+=("${target_test_name_or_null}") +# +#----------------------------------------------------------------------- +# +# If the current item is not a symlink... +# +#----------------------------------------------------------------------- +# + else +# +# Check if the current item is a "regular" file (as defined by bash) and +# thus not a directory or some other exotic entity. If it is a regular +# file, save the corresponding WE2E test name and category subdirectory +# in the arrays prim_test_names and prim_test_subdirs, respectively. +# Also, set its test ID and save it in the array prim_test_ids. If the +# current item is not a regular file, print out a warning and exit. +# + if [ -f "${crnt_item}" ]; then + prim_test_names+=("${test_name_or_null}") + prim_test_subdirs+=("${subdir}") + test_id=$((test_id+1)) + prim_test_ids+=("${test_id}") + else + cwd="$(pwd)" + print_err_msg_exit "\ +The item (crnt_item) in the current directory (cwd) is not a symlink, +but it is also not a \"regular\" file (i.e. it fails bash's -f conditional +operator): + cwd = \"${cwd}\" + crnt_item = \"${crnt_item}\" + [ -f "${crnt_item}" ] = $([ -f "${crnt_item}" ]) +This is probably because it is a directory. Please correct and rerun." + fi + + fi + + fi + + done + + done +# +# For later use, save the number of primary and alternate test names in +# variables. +# + num_prim_tests="${#prim_test_names[@]}" + num_alt_tests="${#alt_test_names[@]}" +# +#----------------------------------------------------------------------- +# +# Create the array test_names that contains both the primary and alternate +# test names found above (with the list of primary names first followed +# by the list of alternate names). Also, create the array test_subdirs +# that contains the category subdirectories corresponding to these test +# names. +# +#----------------------------------------------------------------------- +# + test_names=("${prim_test_names[@]}") + test_subdirs=("${prim_test_subdirs[@]}") + if [ "${num_alt_tests}" -gt "0" ]; then + test_names+=("${alt_test_subdirs[@]:-}") + test_subdirs+=("${alt_test_subdirs[@]:-}") + fi +# +#----------------------------------------------------------------------- +# +# For simplicity, make sure that each test name (either primary or +# alternate) appears exactly once in the array test_names. This is +# equivalent to requiring that a test configuration file (ordinary file +# or symlink) corresponding to each name appear exactly once anywhere +# under the base directory test_configs_basedir. +# +#----------------------------------------------------------------------- +# + num_tests="${#test_names[@]}" + for (( i=0; i<=$((num_tests-1)); i++ )); do + + test_name="${test_names[$i]}" + + subdirs=() + num_occurrences=0 + for (( j=0; j<=$((num_tests-1)); j++ )); do + if [ "${test_names[$j]}" = "${test_name}" ]; then + num_occurrences=$((num_occurrences+1)) + subdirs+=("${test_subdirs[$j]}") + fi + done + + if [ "${num_occurrences}" -ne "1" ]; then + print_err_msg_exit "\ +There must be exactly one WE2E test configuration file (which may be a +ordinary file or a symlink) corresponding to each test name anywhere +under the base directory test_configs_basedir. However, the number of +configuration files (num_occurences) corresponding to the current test +name (test_name) is not 1: + test_configs_basedir = \"${test_configs_basedir}\" + test_name = \"${test_name}\" + num_occurrences = ${num_occurrences} +These configuration files all have the name + \"config.${test_name}.sh\" +and are located in the following category subdirectories under +test_configs_basedir: + subdirs = ( $( printf "\"%s\" " "${subdirs[@]}" )) +Please rename or remove all but one of these configuration files so that +they correspond to unique test names and rerun." + fi + + done +# +#----------------------------------------------------------------------- +# +# If the input argument output_varname_test_descs is not set to a null +# string (meaning that the name of the array in which to return the WE2E +# test descriptions is specified in the call to this function), or if +# the flag generate_csv_file is set to "TRUE", we need to obtain the +# WE2E test descriptions from the test configuration files. In these +# cases, set the local variable get_test_descs to "TRUE". Otherwise, +# set it to "FALSE". +# +#----------------------------------------------------------------------- +# + get_test_descs="FALSE" + if [ ! -z "${output_varname_test_descs}" ] || \ + [ "${generate_csv_file}" = "TRUE" ]; then + get_test_descs="TRUE" + fi +# +#----------------------------------------------------------------------- +# +# If get_test_descs is set to "TRUE", loop through all the primary test +# names and extract from the configuration file of each the description +# of the test. This is assumed to be a section of (bash) comment lines +# at the top of the configuration file. Then append the test description +# to the array prim_test_descs. Note that we assume the first non-comment +# line at the top of the configuration file indicates the end of the test +# description header. +# +#----------------------------------------------------------------------- +# + if [ "${get_test_descs}" = "TRUE" ]; then + + print_info_msg " +Gathering test descriptions from the configuration files of the primary +WE2E tests..." + + prim_test_descs=() + for (( i=0; i<=$((num_prim_tests-1)); i++ )); do + + test_name="${prim_test_names[$i]}" + print_info_msg "\ + Reading in the test description for primary WE2E test: \"${test_name}\"" + subdir=("${prim_test_subdirs[$i]}") + cd_vrfy "${test_configs_basedir}/$subdir" +# +# Keep reading lines from the current test's configuration line until +# a line is encountered that does not start with zero or more spaces, +# followed by the hash symbol (which is the bash comment character) +# possibly followed by a single space character. +# +# In the while-loop below, we read in every such line, strip it of any +# leading spaces, the hash symbol, and possibly another space and append +# what remains to the local variable test_desc. +# + config_fn="config.${test_name}.sh" + test_desc="" + while read -r line; do + + regex_search="^[ ]*(#)([ ]{0,1})(.*)" + hash_or_null=$( printf "%s" "${line}" | \ + sed -n -r -e "s/${regex_search}/\1/p" ) +# +# If the current line is part of the file header containing the test +# description, then... +# + if [ "${hash_or_null}" = "#" ]; then +# +# Strip from the current line any leading whitespace followed by the +# hash symbol possibly followed by a single space. If what remains is +# empty, it means there are no comments on that line and it is just a +# separator line. In that case, simply add a newline to test_desc. +# Otherwise, append what remains after stripping to what test_desc +# already contains, followed by a single space in preparation for +# appending the next (stripped) line. +# + stripped_line=$( printf "%s" "${line}" | \ + sed -n -r -e "s/${regex_search}/\3/p" ) + if [ -z "${stripped_line}" ]; then + test_desc="\ +${test_desc} + +" + else + test_desc="\ +${test_desc}${stripped_line} " + fi +# +# If the current line is not part of the file header containing the test +# description, break out of the while-loop (and thus stop reading the +# file). +# + else + break + fi + + done < "${config_fn}" +# +# At this point, test_desc contains a description of the current test. +# Note that: +# +# 1) It will be empty if the configuration file for the current test +# does not contain a header describing the test. +# 2) It will contain newlines if the description header contained lines +# that start with the hash symbol and contain no other characters. +# These are used to delimit paragraphs within the description. +# 3) It may contain leading and trailing whitespace. +# +# Next, for clarity, we remove any leading and trailing whitespace using +# bash's pattern matching syntax. +# +# Note that the right-hand sides of the following two lines are NOT +# regular expressions. They are expressions that use bash's pattern +# matching syntax (gnu.org/software/bash/manual/html_node/Pattern-Matching.html, +# wiki.bash-hackers.org/syntax/pattern) used in substring removal +# (tldp.org/LDP/abs/html/string-manipulation.html). For example, +# +# ${var%%[![:space:]]*} +# +# says "remove from var its longest substring that starts with a non- +# space character". +# +# First remove leading whitespace. +# + test_desc="${test_desc#"${test_desc%%[![:space:]]*}"}" +# +# Now remove trailing whitespace. +# + test_desc="${test_desc%"${test_desc##*[![:space:]]}"}" +# +# Finally, save the description of the current test as the next element +# of the array prim_test_descs. +# + prim_test_descs+=("${test_desc}") + + done + + fi +# +#----------------------------------------------------------------------- +# +# Create the arrays test_ids and test_descs that initially contain the +# test IDs and descriptions corresponding to the primary test names +# (those of the alternate test names will be appended below). +# +#----------------------------------------------------------------------- +# + test_ids=("${prim_test_ids[@]}") + if [ "${get_test_descs}" = "TRUE" ]; then + test_descs=("${prim_test_descs[@]}") + fi +# +#----------------------------------------------------------------------- +# +# Append to the arrays test_ids and test_descs the test IDs and descriptions +# of the alternate test names. We set the test ID and description of +# each alternate test name to those of the corresponding primary test +# name. +# +#----------------------------------------------------------------------- +# + for (( i=0; i<=$((num_alt_tests-1)); i++ )); do + + alt_test_name="${alt_test_names[$i]}" + alt_test_subdir=("${alt_test_subdirs[$i]}") + target_prim_test_name="${alt_test_prim_test_names[$i]}" + + num_occurrences=0 + for (( j=0; j<=$((num_prim_tests-1)); j++ )); do + if [ "${prim_test_names[$j]}" = "${target_prim_test_name}" ]; then + test_ids+=("${prim_test_ids[$j]}") + if [ "${get_test_descs}" = "TRUE" ]; then + test_descs+=("${prim_test_descs[$j]}") + fi + num_occurrences=$((num_occurrences+1)) + fi + done + + if [ "${num_occurrences}" -ne 1 ]; then + print_err_msg_exit "\ +Each alternate test name must have a corresponding primary test name that +occurs exactly once in the full list of primary test names. For the +current alternate test name (alt_test_name), the number of occurrences +(num_occurrences) of the corresponding primary test name (target_prim_test_name) +is not 1: + alt_test_name = \"${alt_test_name}\" + target_prim_test_name = \"${target_prim_test_name}\" + num_occurrences = \"${num_occurrences}\" +Please correct and rerun." + fi + + done +# +#----------------------------------------------------------------------- +# +# Sort in order of increasing test ID the arrays containing the names, +# IDs, category subdirectories, and descriptions of the WE2E tests. +# +# For this purpose, we first create an array (test_ids_and_inds) each +# of whose elements consist of the test ID, the test type, and the index +# of the array element (with a space used as delimiter). The test type +# is simply an identifier to distinguish between primary test names and +# alternate (symlink-derived) ones. For the former, we set the test +# type to "A", and for the latter, we set it to "B". We do this in order +# to obtain a sorted result in which the elements are not only sorted by +# test ID but also sorted by test type such that within each group of +# elements/tests that has the same test ID, the primary test name is +# listed first followed by zero or more alternte test names. +# +# Next, we sort the array test_ids_and_inds using the "sort" utility +# and save the result in the new array test_ids_and_inds_sorted. The +# latter will be sorted according to test ID because that is the first +# quantity on each line (element) of the original array test_ids_and_inds. +# Also, as described above, for each group of test names that have the +# same ID, the names will be sorted such that the primary test name is +# listed first. +# +# Finally, we extract from test_ids_and_inds_sorted the second number +# in each element (the one afte the first number, which is the test ID, +# and the test type, which we no longer need), which is the original +# array index before sorting, and save the results in the array sort_inds. +# This array will contain the original indices in sorted order that we +# then use to sort the arrays containing the names, IDs, subdirectories, +# and descriptions of the WE2E tests. +# +#----------------------------------------------------------------------- +# + test_ids_and_inds=() + for (( i=0; i<=$((num_tests-1)); i++ )); do + test_type="A" + if [ "$i" -ge "${num_prim_tests}" ]; then + test_type="B" + fi + test_ids_and_inds[$i]="${test_ids[$i]} ${test_type} $i" + done + + readarray -t "test_ids_and_inds_sorted" < \ + <( printf "%s\n" "${test_ids_and_inds[@]}" | sort --numeric-sort ) + + sort_inds=() + regex_search="^[ ]*([0-9]*)[ ]*[AB][ ]*([0-9]*)$" + for (( i=0; i<=$((num_tests-1)); i++ )); do + sort_inds[$i]=$( printf "%s" "${test_ids_and_inds_sorted[$i]}" | \ + sed -n -r -e "s/${regex_search}/\2/p" ) + done + + test_names_orig=( "${test_names[@]}" ) + test_subdirs_orig=( "${test_subdirs[@]}" ) + test_ids_orig=( "${test_ids[@]}" ) + for (( i=0; i<=$((num_tests-1)); i++ )); do + ii="${sort_inds[$i]}" + test_names[$i]="${test_names_orig[$ii]}" + test_subdirs[$i]="${test_subdirs_orig[$ii]}" + test_ids[$i]="${test_ids_orig[$ii]}" + done + + if [ "${get_test_descs}" = "TRUE" ]; then + test_descs_orig=( "${test_descs[@]}" ) + for (( i=0; i<=$((num_tests-1)); i++ )); do + ii="${sort_inds[$i]}" + test_descs[$i]="${test_descs_orig[$ii]}" + done + fi +# +#----------------------------------------------------------------------- +# +# If generate_csv_file is set to "TRUE", generate a CSV (comma-separated +# value) file containing information about the WE2E tests. This file +# can be opened in a spreadsheet in Google Sheets (and possibly Microsoft +# Excel as well) to view information about all the WE2E tests. Note that +# in doing so, the user must specify the field delimiter to be the same +# character that csv_delimiter is set to below. +# +#----------------------------------------------------------------------- +# + if [ "${generate_csv_file}" = "TRUE" ]; then +# +# If a CSV file already exists, delete it. +# + rm_vrfy -f "${csv_fp}" +# +# Set the character used to delimit columns in the CSV file. This has +# to be something that would normally not appear in the fields being +# written to the CSV file. +# + csv_delimiter="|" +# +# Set the titles of the three columns that will be in the file. Then +# write them to the file. The contents of the columns are described in +# more detail further below. +# + column_titles="\ +\"Test Name (Subdirectory)\" ${csv_delimiter} \ +\"Alternate Test Names (Subdirectories)\" ${csv_delimiter} \ +\"Test Purpose/Description\"" + printf "%s\n" "${column_titles}" >> "${csv_fp}" +# +# Loop through the arrays containing the WE2E test information. Extract +# the necessary information and record it to the CSV file row-by-row. +# + j=0 + jp1=$((j+1)) + while [ "$j" -lt "${num_tests}" ]; do +# +# Get the primary name of the test and the category subdirectory in which +# it is located. +# + prim_test_name_subdir="${test_names[$j]} (${test_subdirs[$j]})" +# +# Get the test ID. +# + test_id="${test_ids[$j]}" +# +# Get the test description. +# + test_desc="${test_descs[$j]}" +# +# Replace any double-quotes in the test description with two double-quotes +# since this is the way a double-quote is escaped in a CSV file, at least +# a CSV file that is read in by Google Sheets. +# + test_desc=$( printf "%s" "${test_desc}" | sed -r -e "s/\"/\"\"/g" ) +# +# In the following inner while-loop, we step through all alternate test +# names (if any) that follow the current primary name and construct a +# string (alt_test_names_subdirs) consisting of all the alternate test +# names for this primary name, with each followed by the subdirectory +# the corresponding symlink is in. Note that when the CSV file is opened +# as a spreadsheet (e.g. in Google Sheets), this alternate test name +# information all appears in one cell of the spreadsheet. +# + alt_test_names_subdirs="" + while [ "$jp1" -lt "${num_tests}" ]; do + test_id_next="${test_ids[$jp1]}" + if [ "${test_id_next}" -eq "${test_id}" ]; then + alt_test_names_subdirs="\ +${alt_test_names_subdirs} +${test_names[$jp1]} (${test_subdirs[$jp1]})" + j="$jp1" + jp1=$((j+1)) + else + break + fi + done +# +# Write a line to the CSV file representing a single row of the spreadsheet. +# This row contains the following columns: +# +# Column 1: +# The primary test name followed by the category subdirectory it is +# located in (the latter in parentheses). +# +# Column 2: +# The alternate test names (if any) followed by their subdirectories +# (in parentheses). Each alternate test name and subdirectory pair is +# followed by a newline, but all lines will appear in a single cell of +# the spreadsheet. +# +# Column 3: +# The test description. +# + row_content="\ +\"${prim_test_name_subdir}\" ${csv_delimiter} \ +\"${alt_test_names_subdirs}\" ${csv_delimiter} \ +\"${test_desc}\"" + printf "%s\n" "${row_content}" >> "${csv_fp}" +# +# Update loop indices. +# + j="$jp1" + jp1=$((j+1)) + + done + + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set this function's output variables. Note +# that each of these is set only if the corresponding input variable +# specifying the name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${output_varname_test_configs_basedir}" ]; then + eval ${output_varname_test_configs_basedir}="${test_configs_basedir}" + fi + + if [ ! -z "${output_varname_test_names}" ]; then + test_names_str="( "$( printf "\"%s\" " "${test_names[@]}" )")" + eval ${output_varname_test_names}="${test_names_str}" + fi + + if [ ! -z "${output_varname_test_subdirs}" ]; then + test_subdirs_str="( "$( printf "\"%s\" " "${test_subdirs[@]}" )")" + eval ${output_varname_test_subdirs}="${test_subdirs_str}" + fi + + if [ ! -z "${output_varname_test_ids}" ]; then + test_ids_str="( "$( printf "\"%s\" " "${test_ids[@]}" )")" + eval ${output_varname_test_ids}="${test_ids_str}" + fi + + if [ ! -z "${output_varname_test_descs}" ]; then +# +# We want to treat all characters in the test descriptions literally +# when evaluating the array specified by output_varname_test_descs +# below using the eval function because otherwise, characters such as +# "$", "(", ")", etc will be interpreted as indicating the value of a +# variable, the start of an array, the end of an array, etc, and lead to +# errors. Thus, below, when forming the array that will be passed to +# eval, we will surround each element of the local array test_descs +# in single quotes. However, the test descriptions themselves may +# include single quotes (e.g. when a description contains a phrase such +# as "Please see the User's Guide for..."). In order to treat these +# single quotes literally (as opposed to as delimiters indicating the +# start or end of array elements), we have to pass them as separate +# strings by replacing each single quote with the following series of +# characters: +# +# '"'"' +# +# In this, the first single quote indicates the end of the previous +# single-quoted string, the "'" indicates a string containing a literal +# single quote, and the last single quote inidicates the start of the +# next single-quoted string. +# +# For example, let's assume there are only two WE2E tests to consider. +# Assume the description of the first is +# +# Please see the User's Guide. +# +# and that of the second is: +# +# See description of ${DOT_OR_USCORE} in the configuration file. +# +# Then, if output_varname_test_descs is set to "some_array", the +# exact string we want to pass to eval is: +# +# some_array=('Please see the User'"'"'s Guide.' 'See description of ${DOT_OR_USCORE} in the configuration file.') +# + test_descs_esc_sq=() + for (( i=0; i<=$((num_tests-1)); i++ )); do + test_descs_esc_sq[$i]=$( printf "%s" "${test_descs[$i]}" | \ + sed -r -e "s/'/'\"'\"'/g" ) + done + test_descs_str="( "$( printf "'%s' " "${test_descs_esc_sq[@]}" )")" + eval ${output_varname_test_descs}="${test_descs_str}" + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/tests/get_expts_status.sh b/tests/WE2E/get_expts_status.sh similarity index 100% rename from tests/get_expts_status.sh rename to tests/WE2E/get_expts_status.sh diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh new file mode 100755 index 0000000000..27a23aecb7 --- /dev/null +++ b/tests/WE2E/run_WE2E_tests.sh @@ -0,0 +1,1214 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script runs the specified WE2E tests. Type +# +# run_WE2E_tests.sh --help +# +# for a full description of how to use this script. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script or function is +# located (scrfunc_fp), the name of that file (scrfunc_fn), and the +# directory in which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Set the full path to the top-level directory of the regional_workflow +# repository. We denote this path by homerrfs. The current script +# should be located in the "tests/WE2E" subdirectory under this directory. +# Thus, homerrfs is the directory two levels above the directory in which +# the current script is located. +# +#----------------------------------------------------------------------- +# +homerrfs=${scrfunc_dir%/*/*} +# +#----------------------------------------------------------------------- +# +# Set other directories that depend on homerrfs. +# +#----------------------------------------------------------------------- +# +ushdir="$homerrfs/ush" +testsdir="$homerrfs/tests" +WE2Edir="$testsdir/WE2E" +# +#----------------------------------------------------------------------- +# +# Source bash utility functions. +# +#----------------------------------------------------------------------- +# +. $ushdir/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Source other needed files. +# +#----------------------------------------------------------------------- +# +. ${WE2Edir}/get_WE2Etest_names_subdirs_descs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script or function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Set the usage message. +# +#----------------------------------------------------------------------- +# +usage_str="\ +Usage: + + ${scrfunc_fn} \\ + tests_file=\"...\" \\ + machine=\"...\" \\ + account=\"...\" \\ + [expt_basedir=\"...\"] \\ + [use_cron_to_relaunch=\"...\"] \\ + [cron_relaunch_intvl_mnts=\"...\"] \\ + [verbose=\"...\"] \\ + [stmp=\"...\"] \\ + [ptmp=\"...\"] + +The arguments in brackets are optional. The arguments are defined as +follows: + +tests_file: +Name of file or relative or absolute path to file containing the list of +WE2E tests to run. This file must contain one test name per line, with +no repeated names. This is a required argument. + +machine: +Argument used to explicitly set the experiment variable MACHINE in the +experiment configuration files of all the WE2E tests the user wants to +run. (A description of MACHINE can be found in the default experiment +configuration file.) This is a required argument. + +account: +Argument used to explicitly set the experiment variable ACCOUNT in the +experiment configuration files of all the WE2E tests the user wants to +run. (A description of ACCOUNT can be found in the default experiment +configuration file.) This is a required argument. + +expt_basedir: +Argument used to explicitly set the experiment variable EXPT_BASEDIR in +the experiment configuration files of all the WE2E tests the user wants +to run. (A description of EXPT_BASEDIR can be found in the default +experiment configuration file.) If expt_basedir is specified in the call +to this script, its value is used to set EXPT_BASEDIR in the configuration +files. If it is not specified, EXPT_BASEDIR is not set in the configuration +files, in which case the workflow generation script sets it to a default +value. Note that if expt_basedir is set to a relative path (e.g. +expt_basedir=\"testset1\" in the call to this script), then the workflow +generation script will set EXPT_BASEDIR for the experiment to a default +absolute path followed by \${expt_basedir}. This feature can be used to +group the WE2E tests into subdirectories for convenience, e.g. a set of +tests under subdirectory testset1, another set of tests under testset2, +etc. + +use_cron_to_relaunch: +Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH +in the experiment configuration files of all the WE2E tests the user wants +to run. (A description of USE_CRON_TO_RELAUNCH can be found in the default +experiment configuration file.) If use_cron_to_relaunch is specified in +the call to this script, its value is used to set USE_CRON_TO_RELAUNCH +in the configuration files. If it is not specified, USE_CRON_TO_RELAUNCH +is set to \"TRUE\" in the configuration files, in which case cron jobs +are used to (re)launch the workflows for all tests (one cron job per test). +Thus, use_cron_to_relaunch needs to be specified only if the user wants +to turn off use of cron jobs for all tests (by specifying use_cron_to_relaunch= +\"FALSE\" on the command line). Note that it is not possible to specify +a different value for USE_CRON_TO_RELAUNCH for each test via this argument; +either all tests use cron jobs or none do. + +cron_relaunch_intvl_mnts: +Argument used to explicitly set the experiment variable CRON_RELAUNCH_INTVL_MNTS +in the experiment configuration files of all the WE2E tests the user wants +to run. (A description of CRON_RELAUNCH_INTVL_MNTS can be found in the +default experiment configuration file.) If cron_relaunch_intvl_mnts is +specified in the call to this script, its value is used to set +CRON_RELAUNCH_INTVL_MNTS in the configuration files. If it is not +specified, CRON_RELAUNCH_INTVL_MNTS is set to \"02\" (i.e. two minutes) +in the configuration files. Note that it is not possible to specify a +different value for CRON_RELAUNCH_INTVL_MNTS for each test via this +argument; all tests will use the same value for USE_CRON_TO_RELAUNCH +(either the value specified in the call to this script or the default +value of \"02\"). Note also that the value of this argument matters only +if the argument use_cron_to_relaunch is not explicitly set to \"FALSE\" +in the call to this script. + +verbose: +Argument used to explicitly set the experiment variable VERBOSE in the +experiment configuration files of all the WE2E tests the user wants to +run. (A description of VERBOSE can be found in the default experiment +configuration file.) If verbose is specified in the call to this script, +its value is used to set VERBOSE in the configuration files. If it is +not specified, VERBOSE is set to \"TRUE\" in the configuration files. +Note that it is not possible to specify a different value for VERBOSE +for each test via this argument; either all tests will have VERBOSE set +to \"TRUE\" or all will have it set to \"FALSE\". + +stmp: +Argument used to explicitly set the experiment variable STMP in the +experiment configuration files of all the WE2E tests the user wants to +run that are in NCO mode, i.e. they have test configuration files that +set the experiment variable RUN_ENVIR to \"nco\". (A description of +STMP can be found in the default experiment configuration file.) If +stmp is specified in the call to this script, its value is used to set +STMP in the configuration files of all tests that will run in NCO mode. +If it is not specified, STMP is (effectively) set as follows in the +configuration files (of all NCO mode tests to be run): + + STMP=\$( readlink -f \"\$homerrfs/../../nco_dirs/stmp\" \) + +Here, homerrfs is the base directory in which the regional_workflow +repository is cloned. Note that it is not possible to specify a different +value for STMP for each test via this argument; all tests will use the +same value for STMP (either the value specified in the call to this +script or the default value above). Note also that the value of this +argument is not used for any tests that are not in NCO mode. + +ptmp: +Same as the argument \"stmp\" described above but for setting the +experiment variable PTMP for all tests that will run in NCO mode. +" +# +#----------------------------------------------------------------------- +# +# Check to see if usage help for this script is being requested. If so, +# print it out and exit with a 0 exit code (success). +# +#----------------------------------------------------------------------- +# +help_flag="--help" +if [ "$#" -eq 1 ] && [ "$1" = "${help_flag}" ]; then + print_info_msg "${usage_str}" + exit 0 +fi +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script or function. +# Then process the arguments provided to it on the command line (which +# should consist of a set of name-value pairs of the form arg1="value1", +# arg2="value2", etc). +# +#----------------------------------------------------------------------- +# +valid_args=( \ + "tests_file" \ + "machine" \ + "account" \ + "expt_basedir" \ + "use_cron_to_relaunch" \ + "cron_relaunch_intvl_mnts" \ + "verbose" \ + "stmp" \ + "ptmp" \ + ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# "TRUE". +# +#----------------------------------------------------------------------- +# +print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Verify that the required arguments to this script have been specified. +# If not, print out an error message and exit. +# +#----------------------------------------------------------------------- +# +help_msg="\ +Use + ${scrfunc_fn} ${help_flag} +to get help on how to use this script." + +if [ -z "${tests_file}" ]; then + print_err_msg_exit "\ +The argument \"tests_file\" specifying the file containing a list of the +WE2E tests to run was not specified in the call to this script. \ +${help_msg}" +fi + +if [ -z "${machine}" ]; then + print_err_msg_exit "\ +The argument \"machine\" specifying the machine or platform on which to +run the WE2E tests was not specified in the call to this script. \ +${help_msg}" +fi + +if [ -z "${account}" ]; then + print_err_msg_exit "\ +The argument \"account\" specifying the account under which to submit +jobs to the queue when running the WE2E tests was not specified in the +call to this script. \ +${help_msg}" +fi +# +#----------------------------------------------------------------------- +# +# Get the full path to the file containing the list of user-specified +# WE2E tests to run. Then verify that the file exists. +# +#----------------------------------------------------------------------- +# +user_spec_tests_fp=$( readlink -f "${tests_file}" ) + +if [ ! -f "${user_spec_tests_fp}" ]; then + print_err_msg_exit "\ +The file containing the user-specified list of WE2E tests to run +(tests_file) that is passed in as an argument to this script does not +exit: + tests_file = \"${tests_file}\" +The full path to this script is: + user_spec_tests_fp = \"${user_spec_tests_fp}\" +Please ensure that this file exists and rerun." +fi +# +#----------------------------------------------------------------------- +# +# Read in each line of the file specified by user_spec_tests_fp and add +# each non-empty line to the array user_spec_tests. Note that the read +# command will remove any leading and trailing whitespace from each line +# in user_spec_tests_fp [because it treats whatever character(s) the bash +# variable IFS (Internal Field Separator) is set to as word separators +# on each line, and IFS is by default set to a space, a tab, and a +# newline]. +# +#----------------------------------------------------------------------- +# +user_spec_tests=() +while read -r line; do + if [ ! -z "$line" ]; then + user_spec_tests+=("$line") + fi +done < "${user_spec_tests_fp}" +# +#----------------------------------------------------------------------- +# +# Call a function to obtain the names of all available WE2E tests (i.e. +# not just the ones the user wants to run but all that are part of the +# WE2E testing system), the test IDs, and the category subdirectory in +# which each corresponding test configuration file is located. +# +# The array of test names (avail_WE2E_test_names) that the function +# called below returns contains both primary and alternate test names. +# A primary test name is a test name obtained from the name of a WE2E +# test configuration file that is an ordinary file, i.e. not a symlink, +# whereas an alternate name is one that is derived from the name of a +# symlink whose target is an ordinary test configuration file (but not +# another symlink). To be able to determine the set of test names that +# correspond to the same primary test, the function called also returns +# an array of test IDs (avail_WE2E_test_IDs) such that the IDs for a +# primary test name and all the alternate names that map to it (if any) +# are the same. These IDs will be used later below to ensure that the +# user does not list in the set of test names to run a given test more +# than once, e.g. by accidentally including in the list its primary name +# as well as one of its alternate names. +# +# The category subdirectories in the array avail_WE2E_test_subdirs +# returned by the function called below are relative to the base +# directory under which the WE2E test configuration files are located. +# This base directory is set by the function call below and is returned +# in the output variable avail_WE2E_test_configs_basedir. The i-th +# element of avail_WE2E_test_subdirs specifies the subdirectory under +# this base directory that contains the ordinary test configuration file +# (for a primary test name) or the symlink (for an alternate test name) +# corresponding to the i-th element (which may be a primary or alternate +# test name) in avail_WE2E_test_names. We refer to these subdirectories +# as "category" subdirectories because they are used for clarity to group +# the WE2E tests into types or categories. +# +# Finally, note that the returned arrays +# +# avail_WE2E_test_names +# avail_WE2E_test_ids +# avail_WE2E_test_subdirs +# +# are sorted in order of increasing test ID and such that for a given +# set of test names that share the same ID, the primary test name is +# listed first followed by zero or more alternate names. As an example, +# assume that there are three category subdirectories under the base +# directory specified by avail_WE2E_test_configs_basedir: dir1, dir2, +# and dir3. Also, assume that dir1 contains a test configuration file +# named config.primary_name.sh that is an ordinary file, and dir2 and dir3 +# contain the following symlinks that point config.primary_name.sh: +# +# ${avail_WE2E_test_configs_basedir}/dir2/config.alt_name_1.sh +# --> ${avail_WE2E_test_configs_basedir}/dir1/config.primary_name.sh +# +# ${avail_WE2E_test_configs_basedir}/dir3/config.alt_name_2.sh +# --> ${avail_WE2E_test_configs_basedir}/dir1/config.primary_name.sh +# +# Finally, assume that the ID of the test primary_name is 21 and that +# this ID is at indices 7, 8, and 9 in avail_WE2E_test_ids. Then indices +# 7, 8, and 9 of the three arrays returned by the function call below +# may be as follows: +# +# avail_WE2E_test_names[7]="primary_name" +# avail_WE2E_test_names[8]="alt_name_1" +# avail_WE2E_test_names[9]="alt_name_2" +# +# avail_WE2E_test_ids[7]="21" +# avail_WE2E_test_ids[8]="21" +# avail_WE2E_test_ids[9]="21" +# +# avail_WE2E_test_subdirs[7]="dir1" +# avail_WE2E_test_subdirs[8]="dir2" +# avail_WE2E_test_subdirs[9]="dir3" +# +#----------------------------------------------------------------------- +# +print_info_msg " +Getting information about all available WE2E tests..." + +get_WE2Etest_names_subdirs_descs \ + WE2Edir="${WE2Edir}" \ + output_varname_test_configs_basedir="avail_WE2E_test_configs_basedir" \ + output_varname_test_names="avail_WE2E_test_names" \ + output_varname_test_subdirs="avail_WE2E_test_subdirs" \ + output_varname_test_ids="avail_WE2E_test_ids" +# +# Get the total number of available WE2E test names (including alternate +# names). +# +num_avail_WE2E_tests="${#avail_WE2E_test_names[@]}" +# +#----------------------------------------------------------------------- +# +# Loop through the elements of the array user_spec_tests and perform +# sanity checks. For each such element (i.e. for each WE2E test to run +# specified by the user), make sure that: +# +# 1) The name of the test exists in the complete list of available WE2E +# tests in avail_WE2E_test_names. +# 2) The test does not have an ID that is identical to a previously +# considered test in the user-specified list of tests to run (because +# if so, it would be identical to that previously considered test, +# and it would be a waste of computational resources to run). +# +# If these requirements are met, add the test name to the list of tests +# to run in the array names_tests_to_run, and add the test's category +# subdirectory to subdirs_tests_to_run. +# +#----------------------------------------------------------------------- +# +print_info_msg " +Performing sanity checks on user-specified list of WE2E tests to run..." + +names_tests_to_run=() +ids_tests_to_run=() +subdirs_tests_to_run=() +# +# Initialize the array that will contain the remaining available WE2E +# test names (including alternate names, if any) after finding a match +# for the i-th user-specified test name to run in user_spec_tests. +# +remaining_avail_WE2E_test_names=( "${avail_WE2E_test_names[@]}" ) + +num_user_spec_tests="${#user_spec_tests[@]}" +for (( i=0; i<=$((num_user_spec_tests-1)); i++ )); do + + user_spec_test="${user_spec_tests[$i]}" + + print_info_msg "\ + Checking user-specified WE2E test: \"${user_spec_test}\"" +# +# For the current user-specified WE2E test (user_spec_test), loop through +# the list of all remaining available WE2E test names (i.e. the ones that +# haven't yet been matched to any of the user-specified test names to +# run) and make sure that: +# +# 1) The name of the test exists (either as a primary test name or an +# alternate test name) in the list of all available WE2E test names. +# 2) The test is not repeated in the user-specified list of tests to run, +# either under the same name or an alternate name (i.e. make sure that +# it does not have the same test ID as a previously considered test). +# +# Note that in the loop below, the index j gets set to only those elements +# of remaining_avail_WE2E_test_names that are defined [the syntax +# "${!some_array[@]}" expands to the indices of some_array that have +# defined elements]. We do this for efficiency; we unset elements of +# remaining_avail_WE2E_test_names that have already been matched with +# one of the user-specified test names to run because we know that any +# remaining user-specified test names will not match those elements. +# + match_found="FALSE" + for j in "${!remaining_avail_WE2E_test_names[@]}"; do + + test_name="${avail_WE2E_test_names[$j]}" + test_id="${avail_WE2E_test_ids[$j]}" +# +# Check whether the name of the current user-specified test (user_spec_test) +# matches any of the names in the full list of WE2E tests. If so: +# +# 1) Set match_found to "TRUE". +# 2) Make sure that the test to run doesn't have a test ID that is +# identical to a previously considered test in the user-specified +# list of tests to run (which would mean the two tests are identical). +# If so, print out an error message and exit. +# + if [ "${test_name}" = "${user_spec_test}" ]; then + + match_found="TRUE" + + is_element_of "ids_tests_to_run" "${test_id}" && { + + user_spec_tests_str=$(printf " \"%s\"\n" "${user_spec_tests[@]}") + user_spec_tests_str=$(printf "(\n%s\n )" "${user_spec_tests_str}") + + all_names_for_test=() + for (( k=0; k<=$((num_avail_WE2E_tests-1)); k++ )); do + if [ "${avail_WE2E_test_ids[$k]}" = "${test_id}" ]; then + all_names_for_test+=("${avail_WE2E_test_names[$k]}") + fi + done + all_names_for_test_str=$(printf " \"%s\"\n" "${all_names_for_test[@]}") + + print_err_msg_exit "\ +The current user-specified test to run (user_spec_test) is already included +in the list of tests to run (user_spec_tests), either under the same name +or an alternate name: + user_spec_test = \"${user_spec_test}\" + user_spec_tests = ${user_spec_tests_str} +This test has the following primary and possible alternate names: +${all_names_for_test_str} +In order to avoid repeating the same WE2E test (and thus waste computational +resources), only one of these test names can be specified in the list of +tests to run. Please modify this list in the file + user_spec_tests_fp = \"${user_spec_tests_fp}\" +accordingly and rerun." + + } +# +# Append the name of the current user-specified test, its ID, and its +# category subdirectory to the arrays that contain the sanity-checked +# versions of of these quantities. +# + names_tests_to_run+=("${user_spec_test}") + ids_tests_to_run+=("${test_id}") + subdirs_tests_to_run+=("${avail_WE2E_test_subdirs[$j]}") +# +# Remove the j-th element of remaining_avail_WE2E_test_names so that for +# the next user-specified test to run, we do not need to check whether +# the j-th test is a match. Then break out of the loop over all remaining +# available WE2E tests. +# + unset remaining_avail_WE2E_test_names[$j] + break + + fi + + done +# +# If match_found is still "FALSE" after exiting the loop above, then a +# match for the current user-specifed test to run was not found in the +# list of all WE2E tests -- neither as a primary test name nor as an +# alternate name. In this case, print out an error message and exit. +# + if [ "${match_found}" = "FALSE" ]; then + avail_WE2E_test_names_str=$( printf " \"%s\"\n" "${avail_WE2E_test_names[@]}" ) + print_err_msg_exit "\ +The name current user-specified test to run (user_spec_test) does not +match any of the names (either primary or alternate) of the available +WE2E tests: + user_spec_test = \"${user_spec_test}\" +Valid values for user_spec_test consist of the names (primary or alternate) +of the available WE2E tests, which are: +${avail_WE2E_test_names_str} +Each name in the user-specified list of tests to run: + 1) Must match one of the (primary or alternate) test names of the + availabe WE2E tests. + 2) Must not be the primary or alternate name of a test that has its + primary or one of its alternate names already included in the user- + specified list of test to run, i.e. tests must not be repeated (in + order not to waste computational resources). +Please modify the user-specified list of tests to run such that it adheres +to the rules above and rerun. This list is in the file specified by the +input variable tests_file: + tests_file = \"${tests_file}\" +The full path to this file is: + user_spec_tests_fp = \"${user_spec_tests_fp}\"" + fi + +done +# +#----------------------------------------------------------------------- +# +# Get the number of WE2E tests to run and print out an informational +# message. +# +#----------------------------------------------------------------------- +# +num_tests_to_run="${#names_tests_to_run[@]}" +tests_to_run_str=$( printf " \'%s\'\n" "${names_tests_to_run[@]}" ) +print_info_msg " +After processing the user-specified list of WE2E tests to run, the number +of tests to run (num_tests_to_run) is + num_tests_to_run = ${num_tests_to_run} +and the list of WE2E tests to run (one test per line) is +${tests_to_run_str}" +# +#----------------------------------------------------------------------- +# +# Loop through the WE2E tests to run. For each test, use the corresponding +# test configuration file to generate a temporary experiment file and +# launch the experiment generation script using that file. +# +#----------------------------------------------------------------------- +# +for (( i=0; i<=$((num_tests_to_run-1)); i++ )); do + + test_name="${names_tests_to_run[$i]}" + test_subdir="${subdirs_tests_to_run[$i]}" +# +# Generate the full path to the current WE2E test's configuration file. +# Then ensure that this file exists. +# + test_config_fp="${avail_WE2E_test_configs_basedir}/${test_subdir}/config.${test_name}.sh" + + if [ ! -f "${test_config_fp}" ]; then + print_err_msg_exit "\ +The experiment configuration file (test_config_fp) for the current WE2E +test (test_name) does not exist: + test_name = \"${test_name}\" + test_config_fp = \"${test_config_fp}\" +Please correct and rerun." + fi +# +#----------------------------------------------------------------------- +# +# Source the default experiment configuration file to set values of +# various experiment variables to their defaults. Then source the +# current WE2E test's configuration file to overwrite certain variables' +# default values with test-specific ones. +# +#----------------------------------------------------------------------- +# + . ${ushdir}/config_defaults.sh + . ${test_config_fp} +# +#----------------------------------------------------------------------- +# +# We will now construct a multiline variable consisting of the contents +# that we want the experiment configuration file for this WE2E test to +# have. Once this variable is constructed, we will write its contents +# to the generic configuration file that the experiment generation script +# reads in (specified by the variable EXPT_CONFIG_FN in the default +# configuration file config_defaults.sh sourced above) and then run that +# script to generate an experiment for the current WE2E test. +# +# We name the multiline variable that will contain the contents of the +# experiment configuration file "expt_config_str" (short for "experiment +# configuration string"). Here, we initialize this to a null string, +# and we append to it later below. +# +#----------------------------------------------------------------------- +# + expt_config_str="" +# +#----------------------------------------------------------------------- +# +# Set (and then write to expt_config_str) various experiment variables +# that depend on the input arguments to this script (as opposed to +# variable settings in the test configuration file specified by +# test_config_fp). Note that any values of these parameters specified +# in the default experiment configuration file (config_defaults.sh) +# or in the test configuraiton file (test_config_fp) that were sourced +# above will be overwritten by the settings below. +# +# Note also that if EXPT_BASEDIR ends up getting set to a null string, +# the experiment generation script that gets called further below will +# set it to a default path; if it gets set to a relative path, then the +# experiment generation script will set it to a path consisting of a +# default path with the relative path appended to it; and if it gets set +# to an absolute path, then the workflow will leave it set to that path. +# +#----------------------------------------------------------------------- +# + MACHINE="${machine^^}" + ACCOUNT="${account}" + + EXPT_BASEDIR="${expt_basedir}" + EXPT_SUBDIR="${test_name}" + USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} + CRON_RELAUNCH_INTVL_MNTS=${cron_relaunch_intvl_mnts:-"02"} + VERBOSE=${verbose:-"TRUE"} + + expt_config_str=${expt_config_str}"\ +# +# The machine on which to run, the account to which to charge computational +# resources, the base directory in which to create the experiment directory +# (if different from the default location), and the name of the experiment +# subdirectory. +# +MACHINE=\"${MACHINE}\" +ACCOUNT=\"${ACCOUNT}\"" + + if [ ! -z "${EXPT_BASEDIR}" ]; then + expt_config_str=${expt_config_str}" +EXPT_BASEDIR=\"${EXPT_BASEDIR}\"" + fi + + expt_config_str=${expt_config_str}" +EXPT_SUBDIR=\"${EXPT_SUBDIR}\" +# +# Flag specifying whether or not to automatically resubmit the worfklow +# to the batch system via cron and, if so, the frequency (in minutes) of +# resubmission. +# +USE_CRON_TO_RELAUNCH=\"${USE_CRON_TO_RELAUNCH}\" +CRON_RELAUNCH_INTVL_MNTS=\"${CRON_RELAUNCH_INTVL_MNTS}\" +# +# Flag specifying whether to run in verbose mode. +# +VERBOSE=\"${VERBOSE}\"" +# +#----------------------------------------------------------------------- +# +# Append the contents of the current WE2E test's configuration file to +# the experiment configuration string. +# +#----------------------------------------------------------------------- +# + expt_config_str=${expt_config_str}" +# +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# The following section is a copy of this WE2E test's configuration file. +# +" + expt_config_str=${expt_config_str}$( cat "${test_config_fp}" ) + expt_config_str=${expt_config_str}" +# +# End of section from this test's configuration file. +#----------------------------------------------------------------------- +#-----------------------------------------------------------------------" +# +#----------------------------------------------------------------------- +# +# Modifications to the experiment configuration file if the WE2E test +# uses pre-generated grid, orography, or surface climatology files. +# +# If not running one or more of the grid, orography, and surface +# climatology file generation tasks, specify directories in which +# pregenerated versions of these files can be found. +# +#----------------------------------------------------------------------- +# + if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ] || \ + [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ] || \ + [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then + + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + pregen_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + pregen_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "HERA" ]; then + pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" + elif [ "$MACHINE" = "JET" ]; then + pregen_basedir="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" + elif [ "$MACHINE" = "CHEYENNE" ]; then + pregen_basedir="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" + else + print_err_msg_exit "\ +The base directory (pregen_basedir) in which the pregenerated grid, +orography, and/or surface climatology files are located has not been +specified for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + pregen_dir="${pregen_basedir}/${PREDEF_GRID_NAME}" + + fi +# +# Directory for pregenerated grid files. +# + if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then + GRID_DIR="${pregen_dir}" + expt_config_str=${expt_config_str}" +# +# Directory containing the pregenerated grid files. +# +GRID_DIR=\"${GRID_DIR}\"" + fi +# +# Directory for pregenerated orography files. +# + if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then + OROG_DIR="${pregen_dir}" + expt_config_str=${expt_config_str}" +# +# Directory containing the pregenerated orography files. +# +OROG_DIR=\"${OROG_DIR}\"" + fi +# +# Directory for pregenerated surface climatology files. +# + if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then + SFC_CLIMO_DIR="${pregen_dir}" + expt_config_str=${expt_config_str}" +# +# Directory containing the pregenerated surface climatology files. +# +SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" + fi +# +#----------------------------------------------------------------------- +# +# Modifications to the experiment configuration file if running the WE2E +# test in NCO mode. +# +#----------------------------------------------------------------------- +# + if [ "${RUN_ENVIR}" = "nco" ]; then +# +# Set RUN and envir. +# + expt_config_str=${expt_config_str}" +# +# In order to prevent simultaneous WE2E (Workflow End-to-End) tests that +# are running in NCO mode and which run the same cycles from interfering +# with each other, for each cycle, each such test must have a distinct +# path to the following two directories: +# +# 1) The directory in which the cycle-dependent model input files, symlinks +# to cycle-independent input files, and raw (i.e. before post-processing) +# forecast output files for a given cycle are stored. The path to this +# directory is +# +# \$STMP/tmpnwprd/\$RUN/\$cdate +# +# where cdate is the starting year (yyyy), month (mm), day (dd) and +# hour of the cycle in the form yyyymmddhh. +# +# 2) The directory in which the output files from the post-processor (UPP) +# for a given cycle are stored. The path to this directory is +# +# \$PTMP/com/\$NET/\$envir/\$RUN.\$yyyymmdd/\$hh +# +# Here, we make the first directory listed above unique to a WE2E test +# by setting RUN to the name of the current test. This will also make +# the second directory unique because it also conains the variable RUN +# in its full path, but if this directory -- or set of directories since +# it involves a set of cycles and forecast hours -- already exists from +# a previous run of the same test, then it is much less confusing to the +# user to first move or delete this set of directories during the workflow +# generation step and then start the experiment (whether we move or delete +# depends on the setting of PREEXISTING_DIR_METHOD). For this purpose, +# it is most convenient to put this set of directories under an umbrella +# directory that has the same name as the experiment. This can be done +# by setting the variable envir to the name of the current test. Since +# as mentiond above we will store this name in RUN, below we simply set +# envir to the same value as RUN (which is just EXPT_SUBDIR). Then, for +# this test, the UPP output will be located in the directory +# +# \$PTMP/com/\$NET/\$RUN/\$RUN.\$yyyymmdd/\$hh +# +RUN=\"\${EXPT_SUBDIR}\" +envir=\"\${EXPT_SUBDIR}\"" +# +# Set FIXLAM_NCO_BASEDIR. +# + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + FIXLAM_NCO_BASEDIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + FIXLAM_NCO_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" + elif [ "$MACHINE" = "HERA" ]; then + FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" + elif [ "$MACHINE" = "JET" ]; then + FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" + elif [ "$MACHINE" = "CHEYENNE" ]; then + FIXLAM_NCO_BASEDIR="/needs/to/be/specified" + else + print_err_msg_exit "\ +The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, +orography, and surface climatology \"fixed\" files used in NCO mode are +located has not been specified for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + expt_config_str=${expt_config_str}" +# +# The base directory in which the pregenerated grid, orography, and surface +# climatology \"fixed\" files used in NCO mode are located. In NCO mode, +# the workflow scripts will create symlinks (in the directory specified +# by FIXLAM) to files in a subdirectory under FIXLAM_NCO_BASDEDIR, where +# the name of the subdirectory is the name of the predefined grid specified +# by PREDEF_GRID_NAME. +# +FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" +# +# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model +# for ICs or LBCs. +# + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then + + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + COMINgfs="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" + elif [ "$MACHINE" = "HERA" ]; then + COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" + elif [ "$MACHINE" = "JET" ]; then + COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" + elif [ "$MACHINE" = "CHEYENNE" ]; then + COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" + else + print_err_msg_exit "\ +The directory (COMINgfs) that needs to be specified when running the +workflow in NCO mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or +the GSMGFS as the external model for ICs and/or LBCs has not been specified +for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + expt_config_str=${expt_config_str}" +# +# Directory that needs to be specified when running the workflow in NCO +# mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as +# the external model for ICs and/or LBCs. +# +COMINgfs=\"${COMINgfs}\"" + + fi +# +# Set STMP and PTMP. +# + nco_basedir=$( readlink -f "$homerrfs/../../nco_dirs" ) + STMP=${stmp:-"${nco_basedir}/stmp"} + PTMP=${ptmp:-"${nco_basedir}/ptmp"} + + expt_config_str=${expt_config_str}" +# +# Directories STMP and PTMP that need to be specified when running the +# workflow in NCO-mode (i.e. RUN_ENVIR set to "nco"). +# +STMP=\"${STMP}\" +PTMP=\"${PTMP}\"" + + fi +# +#----------------------------------------------------------------------- +# +# Modifications to the experiment configuration file if the WE2E test +# uses user-staged external model files. +# +#----------------------------------------------------------------------- +# + if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then + + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + extrn_mdl_source_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + extrn_mdl_source_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "HERA" ]; then + extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" + elif [ "$MACHINE" = "JET" ]; then + extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" + elif [ "$MACHINE" = "CHEYENNE" ]; then + extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" + elif [ "$MACHINE" = "ORION" ]; then + extrn_mdl_source_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" + else + print_err_msg_exit "\ +The base directory (extrn_mdl_source_basedir) in which the user-staged +external model files should be located has not been specified for this +machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then + if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then + EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) + elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then + EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) + fi + elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ + [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then + EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) + elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then + EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) + fi + + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" +# +# Make sure that the forecast length is evenly divisible by the interval +# between the times at which the lateral boundary conditions will be +# specified. +# + rem=$(( 10#${FCST_LEN_HRS} % 10#${LBC_SPEC_INTVL_HRS} )) + if [ "$rem" -ne "0" ]; then + print_err_msg_exit "\ +The forecast length (FCST_LEN_HRS) must be evenly divisible by the lateral +boundary conditions specification interval (LBC_SPEC_INTVL_HRS): + FCST_LEN_HRS = ${FCST_LEN_HRS} + LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} + rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" + fi + lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) + EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) + fi + elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) + elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then + EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) + fi + + expt_config_str=${expt_config_str}" +# +# Locations and names of user-staged external model files for generating +# ICs and LBCs. +# +EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" +EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) +EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" +EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" + + fi +# +#----------------------------------------------------------------------- +# +# Set MET and MET+ paths, if necessary. +# +#----------------------------------------------------------------------- +# + if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then + + if [ "$MACHINE" = "HERA" ]; then + met_install_dir="/contrib/met/10.0.0" + metplus_path="/contrib/METplus/METplus-4.0.0" + else + print_err_msg_exit "\ +The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) have not +been specified for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" + fi + + expt_config_str=${expt_config_str}" +# +# MET and MET+ paths. +# +METPLUS_PATH=\"${metplus_path}\" +MET_INSTALL_DIR=\"${met_install_dir}\"" + + fi +# +#----------------------------------------------------------------------- +# +# On some machines (e.g. cheyenne), some tasks often require multiple +# tries before they succeed. To make it more convenient to run the WE2E +# tests on these machines without manual intervention, change the number +# of attempts for such tasks on those machines to be more than one. +# +#----------------------------------------------------------------------- +# + add_maxtries="FALSE" + + if [ "$MACHINE" = "HERA" ]; then + add_maxtries="TRUE" + MAXTRIES_MAKE_ICS="2" + MAXTRIES_MAKE_LBCS="2" + MAXTRIES_RUN_POST="2" + elif [ "$MACHINE" = "CHEYENNE" ]; then + add_maxtries="TRUE" + MAXTRIES_MAKE_SFC_CLIMO="3" + MAXTRIES_MAKE_ICS="5" + MAXTRIES_MAKE_LBCS="10" + MAXTRIES_RUN_POST="10" + fi + + if [ "${add_maxtries}" = "TRUE" ]; then + + expt_config_str=${expt_config_str}" +# +# Maximum number of attempts at running each task. +# +MAXTRIES_MAKE_GRID=\"${MAXTRIES_MAKE_GRID}\" +MAXTRIES_MAKE_OROG=\"${MAXTRIES_MAKE_OROG}\" +MAXTRIES_MAKE_SFC_CLIMO=\"${MAXTRIES_MAKE_SFC_CLIMO}\" +MAXTRIES_GET_EXTRN_ICS=\"${MAXTRIES_GET_EXTRN_ICS}\" +MAXTRIES_GET_EXTRN_LBCS=\"${MAXTRIES_GET_EXTRN_LBCS}\" +MAXTRIES_MAKE_ICS=\"${MAXTRIES_MAKE_ICS}\" +MAXTRIES_MAKE_LBCS=\"${MAXTRIES_MAKE_LBCS}\" +MAXTRIES_RUN_FCST=\"${MAXTRIES_RUN_FCST}\" +MAXTRIES_RUN_POST=\"${MAXTRIES_RUN_POST}\"" + + fi +# +#----------------------------------------------------------------------- +# +# Set the full path to the configuration file that the experiment +# generation script reads in. Then write the contents of expt_config_str +# to that file. +# +#----------------------------------------------------------------------- +# + expt_config_fp="$ushdir/${EXPT_CONFIG_FN}" + printf "%s" "${expt_config_str}" > "${expt_config_fp}" +# +#----------------------------------------------------------------------- +# +# The following are changes that need to be made directly to the +# experiment configuration file created above (as opposed to the +# experiment configuration string expt_config_str) because they involve +# resetting of values that have already been set in the experiment +# configuration file. +# +# If EXTRN_MDL_SYSBASEDIR_ICS has been specified in the current WE2E +# test's base configuration file, it must be set to one of the following: +# +# 1) The string "set_to_non_default_location_in_testing_script" in order +# to allow this script to set it to a valid location depending on the +# machine and external model (for ICs). +# +# 2) To an existing directory. If it is set to a directory, then this +# script ensures that the directory exists (via the check below). +# +#----------------------------------------------------------------------- +# + if [ ! -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then + + if [ "${EXTRN_MDL_SYSBASEDIR_ICS}" = "set_to_non_default_location_in_testing_script" ]; then + + EXTRN_MDL_SYSBASEDIR_ICS="" + if [ "$MACHINE" = "HERA" ]; then + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then + EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" + fi + fi + + if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then + print_err_msg_exit "\ +A non-default location for EXTRN_MDL_SYSBASEDIR_ICS for testing purposes +has not been specified for this machine (MACHINE) and external model for +initial conditions (EXTRN_MDL_NAME_ICS) combination: + MACHINE= \"${MACHINE}\" + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + fi + + else + + if [ ! -d "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then + print_err_msg_exit "\ +The non-default location specified by EXTRN_MDL_SYSBASEDIR_ICS does not +exist or is not a directory: + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + fi + + fi + + set_bash_param "${expt_config_fp}" \ + "EXTRN_MDL_SYSBASEDIR_ICS" "${EXTRN_MDL_SYSBASEDIR_ICS}" + + fi +# +#----------------------------------------------------------------------- +# +# Same as above but for EXTRN_MDL_SYSBASEDIR_LBCS. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then + + if [ "${EXTRN_MDL_SYSBASEDIR_LBCS}" = "set_to_non_default_location_in_testing_script" ]; then + + EXTRN_MDL_SYSBASEDIR_LBCS="" + if [ "$MACHINE" = "HERA" ]; then + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" + fi + fi + + if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then + print_err_msg_exit "\ +A non-default location for EXTRN_MDL_SYSBASEDIR_LBCS for testing purposes +has not been specified for this machine (MACHINE) and external model for +initial conditions (EXTRN_MDL_NAME_LBCS) combination: + MACHINE= \"${MACHINE}\" + EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + fi + + else + + if [ ! -d "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then + print_err_msg_exit "\ +The non-default location specified by EXTRN_MDL_SYSBASEDIR_LBCS does not +exist or is not a directory: + EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + fi + + fi + + set_bash_param "${expt_config_fp}" \ + "EXTRN_MDL_SYSBASEDIR_LBCS" "${EXTRN_MDL_SYSBASEDIR_LBCS}" + + fi +# +#----------------------------------------------------------------------- +# +# Call the experiment generation script to generate an experiment +# directory and a rocoto workflow XML for the current WE2E test to run. +# +#----------------------------------------------------------------------- +# + $ushdir/generate_FV3LAM_wflow.sh || \ + print_err_msg_exit "\ +Could not generate an experiment for the test specified by test_name: + test_name = \"${test_name}\"" + +done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..34a61a6adf --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the CONUS_25km_GFDLgrid grid (which is a +# GFDLgrid type of grid) using the GFS_v16 suite with ICs and LBCs +# derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..a081731e50 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the CONUS_3km_GFDLgrid grid (which is a +# GFDLgrid type of grid) using the GFS_v16 suite with ICs and LBCs +# derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="CONUS_3km_GFDLgrid" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_GSD_HRRR_AK_50km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh similarity index 63% rename from tests/baseline_configs/config.grid_GSD_HRRR_AK_50km.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index 26f5e77667..18ad96810d 100644 --- a/tests/baseline_configs/config.grid_GSD_HRRR_AK_50km.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -1,21 +1,33 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the GSD_HRRR_AK_50km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the RAP. +# +# Note that this test specifies computational resource parameters for +# the MAKE_ICS_TN, MAKE_LBCS_TN, and RUN_POST_TN rocoto tasks in order +# allow the chgres_cube and UPP codes to complete these tasks successfully +# on this very coarse grid. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="GSD_HRRR_AK_50km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="RAP" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" # # For a coarse grid such as this, the number of MPI processes (= NNODES*PPN) # can't be too large for the make_ics and make_lbcs tasks (both of which diff --git a/tests/baseline_configs/config.grid_RRFS_AK_13km_FV3GFS_FV3GFS.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_AK_13km_FV3GFS_FV3GFS.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 31259cbeb4..7a059e4ffd 100644 --- a/tests/baseline_configs/config.grid_RRFS_AK_13km_FV3GFS_FV3GFS.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_AK_13km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_AK_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_AK_13km_RAP_RAP.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_AK_13km_RAP_RAP.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index 2d577710e5..44642ece92 100644 --- a/tests/baseline_configs/config.grid_RRFS_AK_13km_RAP_RAP.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_AK_13km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_AK_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="RAP" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_AK_3km_FV3GFS_FV3GFS.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_AK_3km_FV3GFS_FV3GFS.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 6745aa5e54..d9d1151b0b 100644 --- a/tests/baseline_configs/config.grid_RRFS_AK_3km_FV3GFS_FV3GFS.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_AK_3km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_AK_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_AK_3km_RAP_RAP.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_AK_3km_RAP_RAP.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index b9f44f8f07..e2da03a3bb 100644 --- a/tests/baseline_configs/config.grid_RRFS_AK_3km_RAP_RAP.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_AK_3km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_AK_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="RAP" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 125d8d8d4d..7c6564a30f 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the GFS_v15p2 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 0df2dd8946..de780b806c 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index a97b4330fe..18a62f18d5 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the GSD_SAR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 53% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 5cb5950e34..bec80f24aa 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the HRRR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="1" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 4f8a01107b..b3b420e003 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -1,18 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the RRFS_v1alpha +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -GRID_GEN_METHOD="ESGgrid" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 33a19cbeb4..3f39ab0f72 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the RRFS_v1beta +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.regional_002.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh similarity index 52% rename from tests/baseline_configs/config.regional_002.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh index da84ad058d..2f7d6ecccf 100644 --- a/tests/baseline_configs/config.regional_002.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_2017_gfdlmp +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp_regional.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh similarity index 55% rename from tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp_regional.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh index fdb978f4e1..b21cfbe2cd 100644 --- a/tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp_regional.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_2017_gfdlmp_regional +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 59c0998858..5a36d59886 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 2314a79f7d..3c6841ddf0 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh index db2edaed2f..a313c8d1dc 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..492a163360 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,31 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# physics suite with ICs derived from the FV3GFS and LBCs derived from +# the RAP. +# +# Note that this test specifies the file format of the FV3GFS external +# model data (from which to generate ICs) to be "grib2" as opposed to +# the default value of "nemsio". +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" diff --git a/tests/baseline_configs/config.suite_FV3_CPT_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh similarity index 52% rename from tests/baseline_configs/config.suite_FV3_CPT_v0.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh index e500d9587a..aae81d6ebf 100644 --- a/tests/baseline_configs/config.suite_FV3_CPT_v0.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the CPT_v0 +# physics suite with ICs and LBCs derived from the GSMGFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_CPT_v0" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.DOT_OR_USCORE.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh similarity index 52% rename from tests/baseline_configs/config.DOT_OR_USCORE.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh index 644c7ba24a..7e5b304acc 100644 --- a/tests/baseline_configs/config.DOT_OR_USCORE.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh @@ -1,20 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_2017_gfdlmp +# physics suite with ICs and LBCs derived from the GSMGFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DOT_OR_USCORE="." +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.suite_FV3_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh similarity index 52% rename from tests/baseline_configs/config.suite_FV3_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh index b055a3cb5e..af3a13440f 100644 --- a/tests/baseline_configs/config.suite_FV3_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 +# physics suite with ICs and LBCs derived from the GSMGFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.suite_FV3_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.suite_FV3_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh index 23bc117c3e..616b0e7447 100644 --- a/tests/baseline_configs/config.suite_FV3_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the GSMGFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh index f1d559b1a5..cc239add0c 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the HRRR. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh index fa2001d51a..0e8ad1d04a 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# physics suite with ICs and LBCs derived from the HRRR. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh index af5a5aec07..f236d2f35e 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the HRRR. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 53% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 4241d9ad59..1bd56f0001 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.suite_FV3_GSD_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh similarity index 50% rename from tests/baseline_configs/config.suite_FV3_GSD_v0.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh index be3f3c5f81..21ab6e7abf 100644 --- a/tests/baseline_configs/config.suite_FV3_GSD_v0.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh @@ -1,18 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GSD_v0 +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_v0" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 53% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 215c218e12..01471df077 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="2" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="2" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 53% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index ccb8ff6862..48f2f4c13c 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -1,18 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1alpha +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -GRID_GEN_METHOD="ESGgrid" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 53% rename from tests/baseline_configs/config.subhourly_post_ensemble_2mems.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 5201f332c8..f5fabd9811 100644 --- a/tests/baseline_configs/config.subhourly_post_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -1,25 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="3" -LBC_SPEC_INTVL_HRS="1" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" -DT_ATMOS="120" - -SUB_HOURLY_POST="TRUE" -DT_SUBHOURLY_POST_MNTS="12" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) -DO_ENSEMBLE="TRUE" -NUM_ENS_MEMBERS="2" +FCST_LEN_HRS="3" +LBC_SPEC_INTVL_HRS="1" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh index 53f6b27774..8a7c1c102e 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR +# physics suite with ICs and LBCs derived from the NAM. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" CYCL_HRS=( "12" ) -EXTRN_MDL_NAME_ICS="NAM" -EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh index 42305f3afe..a91c10151a 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# physics suite with ICs and LBCs derived from the NAM. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" CYCL_HRS=( "12" ) -EXTRN_MDL_NAME_ICS="NAM" -EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh index 81e9fc0bb6..a22466e5b4 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the NAM. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="NAM" +EXTRN_MDL_NAME_LBCS="NAM" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" CYCL_HRS=( "12" ) -EXTRN_MDL_NAME_ICS="NAM" -EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="24" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 8f339e834c..528de40de1 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the GFS_v15p2 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 68f24be258..cd2deecb8b 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..960f4b1860 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the GFS_v15p2 +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 9c2a9a5a0f..35301c4666 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the GSD_SAR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 53% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index cfeb6faecd..56b1ffb8be 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the HRRR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="1" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="1" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 90266a61ec..5637d0b7a4 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -1,18 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the RRFS_v1alpha +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -GRID_GEN_METHOD="ESGgrid" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index dbace760fe..07f66f9ccc 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the RRFS_v1beta +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_NA_13km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_NA_13km.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh index 0c0ecbd052..763945bf8f 100644 --- a/tests/baseline_configs/config.grid_RRFS_NA_13km.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_NA_13km grid using the GSD_v0 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_NA_13km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_v0" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh similarity index 71% rename from tests/baseline_configs/config.grid_RRFS_NA_3km.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index fb4d009a5b..9b4e0da931 100644 --- a/tests/baseline_configs/config.grid_RRFS_NA_3km.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -1,21 +1,32 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_NA_3km grid using the RRFS_v1alpha +# physics suite with ICs and LBCs derived from the FV3GFS. +# +# Note that this test also sets various resource parameters for several +# of the rocoto tasks in order to more efficiently run the code on this +# (very large) grid. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_NA_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" ######################################################################### # The following code/namelist/workflow setting changes are necessary to # diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index e74826bf48..eac0a993a5 100644 --- a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -1,17 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_SUBCONUS_3km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 514ed92e57..3534d5df9e 100644 --- a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_SUBCONUS_3km grid using the GFS_v15p2 +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index c625af54cc..4c2ff9f192 100644 --- a/tests/baseline_configs/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,17 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_SUBCONUS_3km grid using the GSD_SAR +# physics suite with ICs derived from the HRRR and LBCs derived from the +# RAP. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..3431f7cdc3 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the CONUS_25km_GFDLgrid grid (which is a GFDLgrid type +# of grid) using the GFS_v16 physics suite with ICs and LBCs derived from +# the FV3GFS. +# + +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190901" +DATE_LAST_CYCL="20190901" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 41f541daff..9bd42fa735 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_25km grid using the GSD_SAR physics +# suite with ICs derived from the HRRR and LBCs derived from the RAP. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 50% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index c16cf54885..300dcd6664 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_25km grid using the HRRR physics suite +# with ICs derived from the HRRR and LBCs derived from the RAP. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200208" DATE_LAST_CYCL="20200208" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh similarity index 52% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh index d43df7d7ca..d8eeef6c5c 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_3km grid using the GFS_2017_gfdlmp_regional +# physics suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190901" CYCL_HRS=( "18" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh similarity index 51% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 3a7d4f32fe..b0d240892d 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -1,20 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_3km grid using the GFS_v15p2 physics +# suite with ICs and LBCs derived from the FV3GFS. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190901" CYCL_HRS=( "18" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" - -FV3GFS_FILE_FMT_ICS="grib2" -FV3GFS_FILE_FMT_LBCS="grib2" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 1748c9936b..eae6b7401a 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_3km grid using the GSD_SAR physics +# suite with ICs derived from the HRRR and LBCs derived from the RAP. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 50% rename from tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 88d0074519..890395ba8c 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_3km grid using the HRRR physics suite +# with ICs derived from the HRRR and LBCs derived from the RAP. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200208" DATE_LAST_CYCL="20200208" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh similarity index 50% rename from tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 2bda9ffe2c..3770547eff 100644 --- a/tests/baseline_configs/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -1,18 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_SUBCONUS_3km grid using the GSD_SAR physics +# suite with ICs derived from the HRRR and LBCs derived from the RAP. +# + RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.GST_release_public_v1.sh b/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh similarity index 66% rename from tests/baseline_configs/config.GST_release_public_v1.sh rename to tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh index 871115be54..0a5cec5252 100644 --- a/tests/baseline_configs/config.GST_release_public_v1.sh +++ b/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh @@ -1,22 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow successfully completes the +# Graduate Student Test (GST) included in Release 1 of the UFS SRW App. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="48" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20190615" -DATE_LAST_CYCL="20190615" -CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" USE_USER_STAGED_EXTRN_FILES="TRUE" -FV3GFS_FILE_FMT_ICS="grib2" -FV3GFS_FILE_FMT_LBCS="grib2" +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="48" +LBC_SPEC_INTVL_HRS="6" WTIME_RUN_FCST="01:00:00" diff --git a/tests/baseline_configs/config.MET_verification.sh b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh similarity index 96% rename from tests/baseline_configs/config.MET_verification.sh rename to tests/WE2E/test_configs/wflow_features/config.MET_verification.sh index fd5ce114c3..0af0eb203e 100644 --- a/tests/baseline_configs/config.MET_verification.sh +++ b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh @@ -10,26 +10,21 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="36" -LBC_SPEC_INTVL_HRS="6" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190415" DATE_LAST_CYCL="20190415" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" - -WTIME_RUN_FCST="01:00:00" +FCST_LEN_HRS="36" +LBC_SPEC_INTVL_HRS="6" MODEL="FV3_GFS_v15p2_CONUS_25km" - RUN_TASK_VX_GRIDSTAT="TRUE" RUN_TASK_VX_POINTSTAT="TRUE" - -USE_USER_STAGED_EXTRN_FILES="TRUE" - +WTIME_RUN_FCST="01:00:00" diff --git a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh new file mode 100644 index 0000000000..922367c726 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh @@ -0,0 +1,31 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to run ensemble forecasts +# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set +# to "community") with the number of ensemble members (NUM_ENS_MEMBERS) +# set to "008". The leading zeros in "008" should cause the ensemble +# members to be numbered "mem001", "mem002", ..., "mem008" (instead of, +# for instance, "mem1", "mem2", ..., "mem8"). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190702" +CYCL_HRS=( "00" "12" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="008" diff --git a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh new file mode 100644 index 0000000000..c9b68006a7 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh @@ -0,0 +1,35 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to run ensemble forecasts +# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set +# to "community") with the number of ensemble members (NUM_ENS_MEMBERS) +# set to "2". The lack of leading zeros in this "2" should cause the +# ensemble members to be named "mem1" and "mem2" (instead of, for instance, +# "mem01" and "mem02"). +# +# Note also that this test uses two cycle hours ("00" and "12") to test +# the capability of the workflow to run ensemble forecasts for more than +# one cycle hour in community mode. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190702" +CYCL_HRS=( "00" "12" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="2" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh new file mode 100644 index 0000000000..a835466b53 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh @@ -0,0 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS grib2-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the first cycle +# date (2019061200) on which the FV3GFS officially became operational. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" + +DATE_FIRST_CYCL="20190612" +DATE_LAST_CYCL="20190612" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh new file mode 100644 index 0000000000..d7891e6686 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh @@ -0,0 +1,29 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS grib2-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on a cycle date +# (2019101818) that is about halfway between the date (2019061200) on +# which the FV3GFS officially became operational and the date (2020022600) +# on which changes to the FV3GFS output files took effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" + +DATE_FIRST_CYCL="20191018" +DATE_LAST_CYCL="20191018" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh new file mode 100644 index 0000000000..a4bea4ac17 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS grib2-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the last cycle date +# (2020022518) before changes to the FV3GFS output files took effect on +# 2020022600. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" + +DATE_FIRST_CYCL="20200225" +DATE_LAST_CYCL="20200225" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh new file mode 100644 index 0000000000..0f4863a0c4 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS grib2-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the first cycle +# date (2020022600) on which changes to the FV3GFS output files took +# effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" + +DATE_FIRST_CYCL="20200226" +DATE_LAST_CYCL="20200226" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh new file mode 100644 index 0000000000..70e0130eb6 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS grib2-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on a cycle date +# (2021010100) that is several months in the future from the date +# (2020022600) on which changes to the FV3GFS output files took effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" + +DATE_FIRST_CYCL="20210101" +DATE_LAST_CYCL="20210101" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh similarity index 50% rename from tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh rename to tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh index f6837dae91..17a9990c68 100644 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh @@ -1,17 +1,24 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived). +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh new file mode 100644 index 0000000000..63cea83776 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh @@ -0,0 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the first cycle +# date (2019061200) on which the FV3GFS officially became operational. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" + +DATE_FIRST_CYCL="20190612" +DATE_LAST_CYCL="20190612" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh new file mode 100644 index 0000000000..cfac4ab62e --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh @@ -0,0 +1,29 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on a cycle date +# (2019101818) that is about halfway between the date (2019061200) on +# which the FV3GFS officially became operational and the date (2020022600) +# on which changes to the FV3GFS output files took effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" + +DATE_FIRST_CYCL="20191018" +DATE_LAST_CYCL="20191018" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh new file mode 100644 index 0000000000..e80507600d --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the last cycle date +# (2020022518) before changes to the FV3GFS output files took effect on +# 2020022600. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" + +DATE_FIRST_CYCL="20200225" +DATE_LAST_CYCL="20200225" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh new file mode 100644 index 0000000000..873a13bef9 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on the first cycle +# date (2020022600) on which changes to the FV3GFS output files took +# effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" + +DATE_FIRST_CYCL="20200226" +DATE_LAST_CYCL="20200226" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh new file mode 100644 index 0000000000..53421da0f5 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh @@ -0,0 +1,28 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS nemsio-formatted output files generated by the FV3GFS external +# model (from which ICs and LBCs will be derived) on a cycle date +# (2021010100) that is several months in the future from the date +# (2020022600) on which changes to the FV3GFS output files took effect. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="nemsio" + +DATE_FIRST_CYCL="20210101" +DATE_LAST_CYCL="20210101" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh similarity index 59% rename from tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh rename to tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh index 929b91d199..441690e810 100644 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh @@ -1,19 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS netcdf-formatted output files generated by the FV3GFS external +# model. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="netcdf" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="netcdf" DATE_FIRST_CYCL="20210620" DATE_LAST_CYCL="20210620" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="netcdf" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="netcdf" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh similarity index 53% rename from tests/baseline_configs/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh rename to tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh index 07705463fb..8c5c54193a 100644 --- a/tests/baseline_configs/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh @@ -1,17 +1,24 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS files generated by the GSMGFS external model from which ICs and +# LBCs will be derived. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh new file mode 100644 index 0000000000..84e93130f7 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS files generated by the HRRR external model from which ICs (and +# the LBCs at the 0th forecast hour) will be derived and files from the +# RAP external model from which the LBCs (except for the ones at the 0th +# forecast hour) will be derived. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GSD_SAR" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh similarity index 52% rename from tests/baseline_configs/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh rename to tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh index 7d39e2f738..62fafdf86c 100644 --- a/tests/baseline_configs/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh @@ -1,17 +1,24 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS files generated by the RAP external model from which ICs and LBCs +# will be derived. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="RAP" +EXTRN_MDL_NAME_LBCS="RAP" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/baseline_configs/config.user_download_extrn_files.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh similarity index 54% rename from tests/baseline_configs/config.user_download_extrn_files.sh rename to tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh index 56b5271ce5..04efe1fa2c 100644 --- a/tests/baseline_configs/config.user_download_extrn_files.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh @@ -1,20 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOMADS +# nemsio-formatted output files generated by the FV3GFS external model +# (from which ICs and LBCs will be derived). +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" DATE_FIRST_CYCL="20200826" DATE_LAST_CYCL="20200826" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" NOMADS="TRUE" NOMADS_file_type="NEMSIO" diff --git a/tests/baseline_configs/config.inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.inline_post.sh similarity index 99% rename from tests/baseline_configs/config.inline_post.sh rename to tests/WE2E/test_configs/wflow_features/config.inline_post.sh index 1e04453069..e491f9b179 100644 --- a/tests/baseline_configs/config.inline_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.inline_post.sh @@ -1,4 +1,4 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # @@ -10,8 +10,6 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -WRITE_DOPOST="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" @@ -25,4 +23,4 @@ CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" - +WRITE_DOPOST="TRUE" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh new file mode 100644 index 0000000000..3b9c93f986 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh @@ -0,0 +1,35 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to run ensemble forecasts +# (i.e. DO_ENSEMBLE set to "TRUE") in nco mode (i.e. RUN_ENVIR set to +# "nco") with the number of ensemble members (NUM_ENS_MEMBERS) set to +# "2". The lack of leading zeros in this "2" should cause the ensemble +# members to be named "mem1" and "mem2" (instead of, for instance, "mem01" +# and "mem02"). +# +# Note also that this test uses two cycle hours ("12" and "18") to test +# the capability of the workflow to run ensemble forecasts for more than +# one cycle hour in nco mode. +# + +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190901" +DATE_LAST_CYCL="20190902" +CYCL_HRS=( "12" "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="2" diff --git a/tests/baseline_configs/config.nco_inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh similarity index 100% rename from tests/baseline_configs/config.nco_inline_post.sh rename to tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh diff --git a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh new file mode 100644 index 0000000000..b56681f549 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh @@ -0,0 +1,62 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of ESGgrid type. + + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" +# +# Define custom grid. +# +GRID_GEN_METHOD="ESGgrid" + +ESGgrid_LON_CTR="-97.5" +ESGgrid_LAT_CTR="41.25" + +ESGgrid_DELX="25000.0" +ESGgrid_DELY="25000.0" + +ESGgrid_NX="216" +ESGgrid_NY="156" + +ESGgrid_WIDE_HALO_WIDTH="6" + +DT_ATMOS="40" + +LAYOUT_X="8" +LAYOUT_Y="12" +BLOCKSIZE="13" + +QUILTING="TRUE" +if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="200" + WRTCMP_ny="150" + WRTCMP_lon_lwr_left="-122.21414225" + WRTCMP_lat_lwr_left="22.41403305" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" +fi diff --git a/tests/baseline_configs/config.new_GFDLgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh similarity index 63% rename from tests/baseline_configs/config.new_GFDLgrid.sh rename to tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh index 27c8544ffe..d27148de2a 100644 --- a/tests/baseline_configs/config.new_GFDLgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh @@ -1,6 +1,47 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable +# +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# +# to "TRUE" (which is its default value); see the UFS SRW User's Guide +# for a description of this variable. +# +# The difference between this test and the one named +# +# new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE +# +# is that this one uses almost no stretching by setting the workflow +# variable GFDLgrid_STRETCH_FAC very close to 1. Setting it exactly to +# 1 used to cause the workflow to fail because it caused the GFDL grid +# generator to assume a global grid. This bug should be rechecked, e.g. +# by setting GFDLgrid_STRETCH_FAC to exactly 1 below. If the grid +# generation succeeds, then this test can be removed. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" +# +# Define custom grid. +# GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" @@ -36,7 +77,6 @@ LAYOUT_Y="6" BLOCKSIZE="26" QUILTING="TRUE" - if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) @@ -51,16 +91,3 @@ if [ "$QUILTING" = "TRUE" ]; then WRTCMP_dlon="0.24" WRTCMP_dlat="0.24" fi - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh similarity index 77% rename from tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh rename to tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh index a4f0e66852..aef2636f2e 100644 --- a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh @@ -1,6 +1,36 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable +# +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# +# to "FALSE"; see the UFS SRW User's Guide for a description of this +# variable. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" +# +# Define custom grid. +# GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" @@ -34,7 +64,6 @@ LAYOUT_Y="6" BLOCKSIZE="26" QUILTING="TRUE" - if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) @@ -49,16 +78,3 @@ if [ "$QUILTING" = "TRUE" ]; then WRTCMP_dlon="0.24" WRTCMP_dlat="0.24" fi - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh similarity index 76% rename from tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh rename to tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh index d94d0ee822..59ca0925e7 100644 --- a/tests/baseline_configs/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh @@ -1,6 +1,36 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable +# +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# +# to "TRUE" (which is its default value); see the UFS SRW User's Guide +# for a description of this variable. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" +# +# Define custom grid. +# GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" @@ -34,7 +64,6 @@ LAYOUT_Y="6" BLOCKSIZE="26" QUILTING="TRUE" - if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) @@ -49,16 +78,3 @@ if [ "$QUILTING" = "TRUE" ]; then WRTCMP_dlon="0.24" WRTCMP_dlat="0.24" fi - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh b/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh similarity index 66% rename from tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh rename to tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh index 4f4653c25a..fa1d0f06a4 100644 --- a/tests/baseline_configs/config.pregen_grid_orog_sfc_climo.sh +++ b/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh @@ -1,21 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to use pregenerated +# grid, orography, and surface climatology files. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" RUN_TASK_MAKE_GRID="FALSE" RUN_TASK_MAKE_OROG="FALSE" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh new file mode 100644 index 0000000000..979d8e96e9 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh @@ -0,0 +1,39 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the character +# (DOT_OR_USCORE) in the names of the input grid and orography files +# that comes after the C-resolution be set to a user-specified value. +# For example, a grid file may be named +# +# C403${DOT_OR_USCORE}grid.tile7.halo4.nc +# +# where "C403" is the C-resolution for this specific grid and +# ${DOT_OR_USCORE} represents the contents of the workflow variable +# DOT_OR_USCORE (bash syntax). DOT_OR_USCORE is by default set to an +# underscore, but for consistency with the rest of the separators in the +# file name (as well as with the character after the C-resolution in the +# names of the surface climatology files), it should be a "." (a dot). +# The MAKE_GRID_TN and MAKE_OROG_TN tasks will name the grid and orography +# files that they create using this character. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" + +EXTRN_MDL_NAME_ICS="GSMGFS" +EXTRN_MDL_NAME_LBCS="GSMGFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190520" +DATE_LAST_CYCL="20190520" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +DOT_OR_USCORE="." diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh similarity index 55% rename from tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh rename to tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index 65a2322c5f..68fa315378 100644 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -1,21 +1,30 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the forecast +# model's time step (DT_ATMOS), its MPI layout (LAYOUT_X and LAYOUT_Y), +# and its cache block size be set to user-specified values. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -DT_ATMOS="100" -LAYOUT_X="10" -LAYOUT_Y="4" -BLOCKSIZE="35" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GSD_SAR" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DT_ATMOS="100" +LAYOUT_X="10" +LAYOUT_Y="4" +BLOCKSIZE="35" diff --git a/tests/baseline_configs/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh similarity index 64% rename from tests/baseline_configs/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh rename to tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh index d6c898bce9..2ade79ff57 100644 --- a/tests/baseline_configs/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh @@ -1,22 +1,29 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the base +# directories on the system disk in which the external model files are +# located be set to user-specified values. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" DATE_FIRST_CYCL="20210603" DATE_LAST_CYCL="20210603" CYCL_HRS=( "06" ) -FCST_LEN_HRS="06" +FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" -FV3GFS_FILE_FMT_ICS="grib2" -FV3GFS_FILE_FMT_LBCS="grib2" EXTRN_MDL_SYSBASEDIR_ICS="set_to_non_default_location_in_testing_script" EXTRN_MDL_SYSBASEDIR_LBCS="set_to_non_default_location_in_testing_script" diff --git a/tests/baseline_configs/config.restart_interval.sh b/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh similarity index 54% rename from tests/baseline_configs/config.restart_interval.sh rename to tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh index 8953dd1a24..3051cb5a89 100644 --- a/tests/baseline_configs/config.restart_interval.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh @@ -1,18 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the time +# interval (RESTART_INTERVAL) at which restart files are written by the +# forecast model be set to a user-specified value. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) -RESTART_INTERVAL="1" -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +RESTART_INTERVAL="1" diff --git a/tests/baseline_configs/config.subhourly_post.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh similarity index 65% rename from tests/baseline_configs/config.subhourly_post.sh rename to tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh index 3d21999544..3edce24dff 100644 --- a/tests/baseline_configs/config.subhourly_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh @@ -1,20 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to have the model write +# output files and perform post-processing on a sub-hourly time interval. +# + RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="3" -LBC_SPEC_INTVL_HRS="1" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FCST_LEN_HRS="3" +LBC_SPEC_INTVL_HRS="1" DT_ATMOS="120" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh new file mode 100644 index 0000000000..337e997401 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh @@ -0,0 +1,38 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to run ensemble forecasts +# that require the forecast model to write output files and perform post- +# processing on a sub-hourly time interval. +# +# This test is needed in addition to the one named "subhourly_post" +# because in the jinja template file from which the rocoto workflow XML +# is generated, the code changes that were made to add the subhourly +# capability also involved changes to the ensemble capability. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="3" +LBC_SPEC_INTVL_HRS="1" + +DT_ATMOS="120" + +SUB_HOURLY_POST="TRUE" +DT_SUBHOURLY_POST_MNTS="12" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="2" diff --git a/tests/baseline_configs/config.community_ensemble_008mems.sh b/tests/baseline_configs/config.community_ensemble_008mems.sh deleted file mode 100644 index 6d53bd725d..0000000000 --- a/tests/baseline_configs/config.community_ensemble_008mems.sh +++ /dev/null @@ -1,21 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190702" -CYCL_HRS=( "00" "12" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DO_ENSEMBLE="TRUE" -NUM_ENS_MEMBERS="008" diff --git a/tests/baseline_configs/config.community_ensemble_2mems.sh b/tests/baseline_configs/config.community_ensemble_2mems.sh deleted file mode 100644 index a3c2e34f08..0000000000 --- a/tests/baseline_configs/config.community_ensemble_2mems.sh +++ /dev/null @@ -1,21 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190702" -CYCL_HRS=( "00" "12" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DO_ENSEMBLE="TRUE" -NUM_ENS_MEMBERS="2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh deleted file mode 100644 index 6f37bdb94d..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190612" -DATE_LAST_CYCL="20190612" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh deleted file mode 100644 index 2bcdf7a5a8..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20191018" -DATE_LAST_CYCL="20191018" -CYCL_HRS=( "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh deleted file mode 100644 index fe2b59ae3e..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200225" -DATE_LAST_CYCL="20200225" -CYCL_HRS=( "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh deleted file mode 100644 index f2a33611e0..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200226" -DATE_LAST_CYCL="20200226" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh deleted file mode 100644 index 09fde49fa5..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20210101" -DATE_LAST_CYCL="20210101" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh deleted file mode 100644 index ae813d595e..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190612" -DATE_LAST_CYCL="20190612" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh deleted file mode 100644 index 6a95be86c6..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20191018" -DATE_LAST_CYCL="20191018" -CYCL_HRS=( "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh deleted file mode 100644 index 9660605f94..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200225" -DATE_LAST_CYCL="20200225" -CYCL_HRS=( "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh deleted file mode 100644 index c1cdc625b3..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200226" -DATE_LAST_CYCL="20200226" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh b/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh deleted file mode 100644 index 431688b009..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20210101" -DATE_LAST_CYCL="20210101" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_ICS="nemsio" diff --git a/tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh deleted file mode 100644 index 5485cc4760..0000000000 --- a/tests/baseline_configs/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" diff --git a/tests/baseline_configs/config.grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS.sh deleted file mode 100644 index 4b79b6ce54..0000000000 --- a/tests/baseline_configs/config.grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS.sh b/tests/baseline_configs/config.grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS.sh deleted file mode 100644 index bf1928ad80..0000000000 --- a/tests/baseline_configs/config.grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="CONUS_3km_GFDLgrid" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh deleted file mode 100644 index 45975fa2b3..0000000000 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="1" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="RAP" -FV3GFS_FILE_FMT_ICS="grib2" -FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh deleted file mode 100644 index 3bd8b82454..0000000000 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_RRFS_v1beta" -FCST_LEN_HRS="03" -LBC_SPEC_INTVL_HRS="1" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh deleted file mode 100644 index 5ff6cc8230..0000000000 --- a/tests/baseline_configs/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_13km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.nco_ensemble.sh b/tests/baseline_configs/config.nco_ensemble.sh deleted file mode 100644 index 14dab8fb24..0000000000 --- a/tests/baseline_configs/config.nco_ensemble.sh +++ /dev/null @@ -1,21 +0,0 @@ -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190902" -CYCL_HRS=( "12" "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DO_ENSEMBLE="TRUE" -NUM_ENS_MEMBERS="2" diff --git a/tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh deleted file mode 100644 index e83617b61a..0000000000 --- a/tests/baseline_configs/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ /dev/null @@ -1,18 +0,0 @@ -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v16" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190901" -CYCL_HRS=( "18" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.new_ESGgrid.sh b/tests/baseline_configs/config.new_ESGgrid.sh deleted file mode 100644 index 3588253180..0000000000 --- a/tests/baseline_configs/config.new_ESGgrid.sh +++ /dev/null @@ -1,49 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -GRID_GEN_METHOD="ESGgrid" - -ESGgrid_LON_CTR="-97.5" -ESGgrid_LAT_CTR="41.25" - -ESGgrid_DELX="25000.0" -ESGgrid_DELY="25000.0" - -ESGgrid_NX="216" -ESGgrid_NY="156" - -ESGgrid_WIDE_HALO_WIDTH="6" - -DT_ATMOS="40" - -LAYOUT_X="8" -LAYOUT_Y="12" -BLOCKSIZE="13" - -QUILTING="TRUE" -WRTCMP_write_groups="1" -WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) -WRTCMP_output_grid="lambert_conformal" -WRTCMP_cen_lon="${ESGgrid_LON_CTR}" -WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" -WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" -WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" -WRTCMP_nx="200" -WRTCMP_ny="150" -WRTCMP_lon_lwr_left="-122.21414225" -WRTCMP_lat_lwr_left="22.41403305" -WRTCMP_dx="${ESGgrid_DELX}" -WRTCMP_dy="${ESGgrid_DELY}" - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="3" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp.sh b/tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp.sh deleted file mode 100644 index 97dca1cc8e..0000000000 --- a/tests/baseline_configs/config.suite_FV3_GFS_2017_gfdlmp.sh +++ /dev/null @@ -1,18 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_GSD_SAR.sh b/tests/baseline_configs/config.suite_FV3_GSD_SAR.sh deleted file mode 100644 index 51b1c06853..0000000000 --- a/tests/baseline_configs/config.suite_FV3_GSD_SAR.sh +++ /dev/null @@ -1,18 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_HRRR.sh b/tests/baseline_configs/config.suite_FV3_HRRR.sh deleted file mode 100644 index 72a950347e..0000000000 --- a/tests/baseline_configs/config.suite_FV3_HRRR.sh +++ /dev/null @@ -1,17 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_HRRR" -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh b/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh deleted file mode 100644 index 9383604dcd..0000000000 --- a/tests/baseline_configs/config.suite_FV3_RRFS_v1alpha.sh +++ /dev/null @@ -1,19 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -GRID_GEN_METHOD="ESGgrid" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baseline_configs/config.suite_FV3_RRFS_v1beta.sh b/tests/baseline_configs/config.suite_FV3_RRFS_v1beta.sh deleted file mode 100644 index 5e7b2211aa..0000000000 --- a/tests/baseline_configs/config.suite_FV3_RRFS_v1beta.sh +++ /dev/null @@ -1,18 +0,0 @@ -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -QUILTING="TRUE" - -CCPP_PHYS_SUITE="FV3_RRFS_v1beta" - -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/tests/baselines_list.txt b/tests/baselines_list.txt deleted file mode 100644 index 34126824b1..0000000000 --- a/tests/baselines_list.txt +++ /dev/null @@ -1,86 +0,0 @@ -DOT_OR_USCORE -GST_release_public_v1 -community_ensemble_008mems -community_ensemble_2mems -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100 -get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000 -get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS -get_from_HPSS_ics_HRRR_lbcs_RAP -get_from_HPSS_ics_RAP_lbcs_RAP -grid_CONUS_25km_GFDLgrid_FV3GFS_FV3GFS -grid_CONUS_3km_GFDLgrid_FV3GFS_FV3GFS -grid_GSD_HRRR_AK_50km -grid_RRFS_AK_13km_FV3GFS_FV3GFS -grid_RRFS_AK_13km_RAP_RAP -grid_RRFS_AK_3km_FV3GFS_FV3GFS -grid_RRFS_AK_3km_RAP_RAP -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR -grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR -grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_modify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -grid_RRFS_NA_13km -grid_RRFS_NA_3km -grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2 -grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -inline_post -nco_ensemble -nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR -nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional -nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR -nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR -nco_inline_post -new_ESGgrid -new_GFDLgrid -new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE -new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE -pregen_grid_orog_sfc_climo -regional_002 -restart_interval -subhourly_post -subhourly_post_ensemble_2mems -suite_FV3_CPT_v0 -suite_FV3_GFS_2017_gfdlmp -suite_FV3_GFS_2017_gfdlmp_regional -suite_FV3_GFS_v15p2 -suite_FV3_GFS_v16 -suite_FV3_GSD_SAR -suite_FV3_GSD_v0 -suite_FV3_HRRR -suite_FV3_RRFS_v1beta -MET_verification diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh deleted file mode 100755 index 2eb82965ac..0000000000 --- a/tests/run_experiments.sh +++ /dev/null @@ -1,994 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# The current script should be located in the "tests" subdirectory of the -# workflow's top-level directory, which we denote by homerrfs. Thus, -# homerrfs is the directory one level above the directory in which the -# current script is located. Set homerrfs accordingly. -# -#----------------------------------------------------------------------- -# -homerrfs=${scrfunc_dir%/*} -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# -ushdir="$homerrfs/ush" -baseline_configs_dir="$homerrfs/tests/baseline_configs" -# -#----------------------------------------------------------------------- -# -# Source bash utility functions. -# -#----------------------------------------------------------------------- -# -. $ushdir/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. -# Then process the arguments provided to this script/function (which -# should consist of a set of name-value pairs of the form arg1="value1", -# etc). -# -#----------------------------------------------------------------------- -# -valid_args=( \ -"expts_file" \ -"machine" \ -"account" \ -"expt_basedir" \ -"testset_name" \ -"use_cron_to_relaunch" \ -"cron_relaunch_intvl_mnts" \ -"verbose" \ -"stmp" \ -"ptmp" \ -) -process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# -print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# -if [ 1 = 0 ]; then - if [ "$#" -ne 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Number of arguments specified: $# - -Usage: - - ${scrfunc_fn} expts_file - -where expts_file is the name of the file containing the list of experi- -ments to run. If expts_file is the absolute path to a file, it is used -as is. If it is a relative path (including just a file name), it is as- -sumed to be given relative to the path from which this script is called. -" - - fi -fi -# -#----------------------------------------------------------------------- -# -# Verify that an experiments list file has been specified. If not, -# print out an error message and exit. -# -#----------------------------------------------------------------------- -# -# Note: -# The function process_args() should be modified to look for required -# arguments, which can be denoted by appending to the name of a required -# argument the string "; REQUIRED". It can then check that all required -# arguments are in fact specified in the arguments list. That way, the -# following if-statement will not be needed since process_args() will -# catch the case of missing required arguments. -# -if [ -z "${expts_file}" ] || \ - [ -z "${machine}" ] || \ - [ -z "${account}" ]; then - print_err_msg_exit "\ -An experiments list file (expts_file), a machine name (machine), and an -account name (account) must be specified as input arguments to this -script. One or more of these is currently set to an empty string: - expts_file = \"${expts_file}\" - machine = \"${machine}\" - account = \"${account}\" -Use the following format to specify these in the argument list passed to -this script: - ${scrfunc_fn} \\ - expts_file=\"name_of_file_or_full_path_to_file\" \\ - machine=\"name_of_machine_to_run_on\" \\ - account=\"name_of_hpc_account_to_use\" \\ - ..." -fi -# -#----------------------------------------------------------------------- -# -# Get the full path to the experiments list file and verify that it exists. -# -#----------------------------------------------------------------------- -# -expts_list_fp=$( readlink -f "${expts_file}" ) - -if [ ! -f "${expts_list_fp}" ]; then - print_err_msg_exit "\ -The experiments list file (expts_file) specified as an argument to this -script (and with full path given by expts_list_fp) does not exist: - expts_file = \"${expts_file}\" - expts_list_fp = \"${expts_list_fp}\"" -fi -# -#----------------------------------------------------------------------- -# -# Read in the list of experiments (which might be baselines) to run. -# This entails reading in each line of the file expts_list.txt in the -# directory of this script and saving the result in the array variable -# expts_list. Note that each line of expts_list.txt has the form -# -# BASELINE_NAME | VAR_NAME_1="VAR_VALUE_1" | ... | VAR_NAME_N="VAR_VALUE_N" -# -# where BASELINE_NAME is the name of the baseline and the zero or more -# variable name-value pairs following the baseline name are a list of -# variables to modify from the baseline. Note that: -# -# 1) There must exist a experiment/workflow configuration file named -# config.BASELINE_NAME.sh in a subdirectory named baseline_configs -# in the directory of this script. -# -# 2) The variable name-value pairs on each line of the expts_list.txt -# file are delimited from the baseline and from each other by pipe -# characters (i.e. "|"). -# -#----------------------------------------------------------------------- -# -print_info_msg " -Reading in list of forecast experiments from file - expts_list_fp = \"${expts_list_fp}\" -and storing result in the array \"all_lines\" (one array element per expe- -riment)..." - -readarray -t all_lines < "${expts_list_fp}" - -all_lines_str=$( printf "\'%s\'\n" "${all_lines[@]}" ) -print_info_msg " -All lines from experiments list file (expts_list_fp) read in, where: - expts_list_fp = \"${expts_list_fp}\" -Contents of file are (line by line, each line within single quotes, and -before any processing): - -${all_lines_str} -" -# -#----------------------------------------------------------------------- -# -# Loop through the elements of all_lines and modify each line to remove -# leading and trailing whitespace and any whitespace before and after the -# field separator character (which is the pipe character, "|"). Also, -# drop any elements that are empty after this processing, and save the -# resulting set of non-empty elements in the array expts_list. -# -#----------------------------------------------------------------------- -# -expts_list=() -field_separator="\|" # Need backslash as an escape sequence in the sed commands below. - -j=0 -num_lines="${#all_lines[@]}" -for (( i=0; i<=$((num_lines-1)); i++ )); do -# -# Remove all leading and trailing whitespace from the current element of -# all_lines. -# - all_lines[$i]=$( printf "%s" "${all_lines[$i]}" | \ - sed -r -e "s/^[ ]*//" -e "s/[ ]*$//" ) -# -# Remove spaces before and after all field separators in the current -# element of all_lines. Note that we use the pipe symbol, "|", as the -# field separator. -# - all_lines[$i]=$( printf "%s" "${all_lines[$i]}" | \ - sed -r -e "s/[ ]*${field_separator}[ ]*/${field_separator}/g" ) -# -# If the last character of the current line is a field separator, remove -# it. -# - all_lines[$i]=$( printf "%s" "${all_lines[$i]}" | \ - sed -r -e "s/${field_separator}$//g" ) -# -# If after the processing above the current element of all_lines is not -# empty, save it as the next element of expts_list. -# - if [ ! -z "${all_lines[$i]}" ]; then - expts_list[$j]="${all_lines[$i]}" - j=$((j+1)) - fi - -done -# -#----------------------------------------------------------------------- -# -# Get the number of experiments to run and print out an informational -# message. -# -#----------------------------------------------------------------------- -# -num_expts="${#expts_list[@]}" -expts_list_str=$( printf " \'%s\'\n" "${expts_list[@]}" ) -print_info_msg " -After processing, the number of experiments to run (num_expts) is: - num_expts = ${num_expts} -The list of forecast experiments to run (one experiment per line) is gi- -ven by: -${expts_list_str} -" -# -#----------------------------------------------------------------------- -# -# Loop through the elements of the array expts_list. For each element -# (i.e. for each experiment), generate an experiment directory and cor- -# responding workflow and then launch the workflow. -# -#----------------------------------------------------------------------- -# -for (( i=0; i<=$((num_expts-1)); i++ )); do - - print_info_msg " -Processing experiment \"${expts_list[$i]}\" ..." -# -# Get the name of the baseline on which the current experiment is based. -# Then save the remainder of the current element of expts_list in the -# variable "remainder". Note that if this variable is empty, then the -# current experiment is identical to the current baseline. If not, then -# "remainder" contains the modifications that need to be made to the -# current baseline to obtain the current experiment. -# - regex_search="^([^${field_separator}]*)(${field_separator}(.*)|)" - baseline_name=$( printf "%s" "${expts_list[$i]}" | \ - sed -r -n -e "s/${regex_search}/\1/p" ) - remainder=$( printf "%s" "${expts_list[$i]}" | \ - sed -r -n -e "s/${regex_search}/\3/p" ) -# -# Get the names and corresponding values of the variables that need to -# be modified in the current baseline to obtain the current experiment. -# The following while-loop steps through all the variables listed in -# "remainder". -# - modvar_name=() - modvar_value=() - num_mod_vars=0 - while [ ! -z "${remainder}" ]; do -# -# Get the next variable-value pair in remainder, and save what is left -# of remainder back into itself. -# - next_field=$( printf "%s" "$remainder" | \ - sed -r -e "s/${regex_search}/\1/" ) - remainder=$( printf "%s" "$remainder" | \ - sed -r -e "s/${regex_search}/\3/" ) -# -# Save the name of the variable in the variable-value pair obtained -# above in the array modvar_name. Then save the value in the variable- -# value pair in the array modvar_value. -# - modvar_name[${num_mod_vars}]=$( printf "%s" "${next_field}" | \ - sed -r -e "s/^([^=]*)=(.*)/\1/" ) - modvar_value[${num_mod_vars}]=$( printf "%s" "${next_field}" | \ - sed -r -e "s/^([^=]*)=(\")?([^\"]+*)(\")?/\3/" ) -# -# Increment the index that keeps track of the number of variables that -# need to be modified in the current baseline to obtain the current ex- -# periment. -# - num_mod_vars=$((num_mod_vars+1)) - - done -# -# Generate the path to the configuration file for the current baseline. -# This will be modified to obtain the configuration file for the current -# experiment. -# - baseline_config_fp="${baseline_configs_dir}/config.${baseline_name}.sh" -# -# Print out an error message and exit if a configuration file for the -# current baseline does not exist. -# - if [ ! -f "${baseline_config_fp}" ]; then - print_err_msg_exit "\ -The experiment/workflow configuration file (baseline_config_fp) for the -specified baseline (baseline_name) does not exist: - baseline_name = \"${baseline_name}\" - baseline_config_fp = \"${baseline_config_fp}\" -Please correct and rerun." - fi -# -# Generate a name for the current experiment. We start with the name of -# the current baseline and modify it to indicate which variables must be -# reset to obtain the current experiment. -# - expt_name="${baseline_name}" - for (( j=0; j<${num_mod_vars}; j++ )); do - if [ $j -lt ${#modvar_name[@]} ]; then - expt_name="${expt_name}__${modvar_name[$j]}.eq.${modvar_value[$j]}" - else - break - fi - done -# -# Set the full path to the workflow configuration file for the current -# experiment that the workflow generation script will read in. For now, -# include the name of the test in the file name. Once this file is -# constructed below, it will get renamed to the file name that the -# generation script expects (which is "config.sh"). Also, if a preexisting -# file of this name exists, delete it. -# - expt_config_fp="$ushdir/config.${expt_name}.sh" - rm_vrfy -rf "${expt_config_fp}" -# -#----------------------------------------------------------------------- -# -# Source the default workflow configuration file. Note that we need to -# re-source this file for each WE2E test because the previous test may -# change these default values when the test-specific configuration file -# is sourced below. We need to reset the workflow variables because some -# of the tests rely on the default values. -# -#----------------------------------------------------------------------- -# - . ${ushdir}/config_defaults.sh -# -#----------------------------------------------------------------------- -# -# Source the WE2E test configuration file. This will overwrite some of -# the workflow variable values in the default workflow configuration file -# sourced above. -# -#----------------------------------------------------------------------- -# - . ${baseline_config_fp} -# -#----------------------------------------------------------------------- -# -# Set various workflow variables that depend on inputs to this script (as -# opposed to information in the test-specific configuration file specified -# by baseline_config_fp). Note that any values of these parameters -# specified in the default workflow configuration file (config_defaults.sh) -# or in the test-specific configuraiton file (baseline_config_fp) that -# are sourced above will be overwritten by the settings below. -# -# Note that EXPT_BASEDIR is set below as follows: -# * If neither of the command line arguments expt_basedir and testset_name -# to this script are specified, EXPT_BASEDIR gets set to a null string. -# * If expt_basedir is specified but testset_name is not, EXPT_BASEDIR -# gets set to expt_basedir. -# * If expt_basedir is not specified but testset_name is, EXPT_BASEDIR -# gets set to testset_name. -# * If expt_basedir and testset_name are both specified, EXPT_BASEDIR -# gets set to expt_basedir with testset_name appended to it (with a -# "/" in between). -# -# Note also that if EXPT_BASEDIR ends up getting set to a null string, -# the workflow generation script that gets called further below will set -# it to a default path; if it gets set to a relative path, then the workflow -# generation script will set it to a path consisting of a default path -# with the relative path appended to it; and if it gets set to an absolute -# path, then the workflow will leave it set to that path. -# -#----------------------------------------------------------------------- -# - MACHINE="${machine^^}" - ACCOUNT="${account}" - -# Note that if expt_basedir is a null (or unset) string, ${expt_basedir:+/} -# gets set to a null string; otherwise, it gets set to "/". - EXPT_BASEDIR="${expt_basedir}${expt_basedir:+/}${testset_name}" -# Remove any trailing "/" from EXPT_BASEDIR. - EXPT_BASEDIR="${EXPT_BASEDIR%%/}" - - EXPT_SUBDIR="${expt_name}" - USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} - CRON_RELAUNCH_INTVL_MNTS=${cron_relaunch_intvl_mnts:-"02"} - VERBOSE=${verbose:-"TRUE"} - - str="\ -# -# The machine on which to run, the account to which to charge computational -# resources, the base directory in which to create the experiment directory -# (if different from the default location), and the name of the experiment -# subdirectory. -# -MACHINE=\"${MACHINE}\" -ACCOUNT=\"${ACCOUNT}\"" - - if [ ! -z "${EXPT_BASEDIR}" ]; then - str=${str}" -EXPT_BASEDIR=\"${EXPT_BASEDIR}\"" - fi - - str=${str}" -EXPT_SUBDIR=\"${EXPT_SUBDIR}\" -# -# Flag specifying whether or not to automatically resubmit the worfklow -# to the batch system via cron and, if so, the frequency (in minutes) of -# resubmission. -# -USE_CRON_TO_RELAUNCH=\"${USE_CRON_TO_RELAUNCH}\" -CRON_RELAUNCH_INTVL_MNTS=\"${CRON_RELAUNCH_INTVL_MNTS}\" -# -# Flag specifying whether to run in verbose mode. -# -VERBOSE=\"${VERBOSE}\"" -# -#----------------------------------------------------------------------- -# -# Append test-specific values to the workflow configuration file. -# -#----------------------------------------------------------------------- -# - str=${str}" -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# The following section is a copy of the base configuration of this WE2E -# test. -# -" - str=${str}$( cat "${baseline_config_fp}" ) - str=${str}" -# -# End of section from the base configuration file of this WE2E test. -#----------------------------------------------------------------------- -#-----------------------------------------------------------------------" -# -#----------------------------------------------------------------------- -# -# If not running one or more of the grid, orography, and surface climatology -# file generation tasks, specify directories in which pregenerated files -# can be found. -# -#----------------------------------------------------------------------- -# - if [ ${RUN_TASK_MAKE_GRID} = "FALSE" ] || \ - [ ${RUN_TASK_MAKE_OROG} = "FALSE" ] || \ - [ ${RUN_TASK_MAKE_SFC_CLIMO} = "FALSE" ]; then - -# Note: -# Now that the "grid", "orog", and "sfc_climo" sub-subdirectories under -# pregen_basedir have been removed, we don't need the variable pregen_basedir -# and can instead have the variable "pregen_dir" that gets set to -# ${pregen_basedir}/${PREDEF_GRID_NAME}, and pregen_dir can then be used -# to set GRID_DIR, OROG_DIR, and/or SFC_CLIMO_DIR below. - - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - pregen_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - pregen_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "HERA" ]; then - pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" - elif [ "$MACHINE" = "JET" ]; then - pregen_basedir="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" - elif [ "$MACHINE" = "CHEYENNE" ]; then - pregen_basedir="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" - else - print_err_msg_exit "\ -The base directory (pregen_basedir) in which the pregenerated grid, -orography, and/or surface climatology files are located has not been -specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - fi -# -# Directory for pregenerated grid files. -# - if [ ${RUN_TASK_MAKE_GRID} = "FALSE" ]; then - GRID_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" - str=${str}" -# -# Directory containing the pregenerated grid files. -# -GRID_DIR=\"${GRID_DIR}\"" - - fi -# -# Directory for pregenerated orography files. -# - if [ ${RUN_TASK_MAKE_OROG} = "FALSE" ]; then - OROG_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" - str=${str}" -# -# Directory containing the pregenerated orography files. -# -OROG_DIR=\"${OROG_DIR}\"" - - fi -# -# Directory for pregenerated surface climatology files. -# - if [ ${RUN_TASK_MAKE_SFC_CLIMO} = "FALSE" ]; then - SFC_CLIMO_DIR="${pregen_basedir}/${PREDEF_GRID_NAME}" - str=${str}" -# -# Directory containing the pregenerated surface climatology files. -# -SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" - - fi -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# - if [ "${RUN_ENVIR}" = "nco" ]; then -# -# Set RUN and envir. -# - str=${str}" -# -# In order to prevent simultaneous WE2E (Workflow End-to-End) tests that -# are running in NCO mode and which run the same cycles from interfering -# with each other, for each cycle, each such test must have a distinct -# path to the following two directories: -# -# 1) The directory in which the cycle-dependent model input files, symlinks -# to cycle-independent input files, and raw (i.e. before post-processing) -# forecast output files for a given cycle are stored. The path to this -# directory is -# -# \$STMP/tmpnwprd/\$RUN/\$cdate -# -# where cdate is the starting year (yyyy), month (mm), day (dd) and -# hour of the cycle in the form yyyymmddhh. -# -# 2) The directory in which the output files from the post-processor (UPP) -# for a given cycle are stored. The path to this directory is -# -# \$PTMP/com/\$NET/\$envir/\$RUN.\$yyyymmdd/\$hh -# -# Here, we make the first directory listed above unique to a WE2E test -# by setting RUN to the name of the current test. This will also make -# the second directory unique because it also conains the variable RUN -# in its full path, but if this directory -- or set of directories since -# it involves a set of cycles and forecast hours -- already exists from -# a previous run of the same test, then it is much less confusing to the -# user to first move or delete this set of directories during the workflow -# generation step and then start the experiment (whether we move or delete -# depends on the setting of PREEXISTING_DIR_METHOD). For this purpose, -# it is most convenient to put this set of directories under an umbrella -# directory that has the same name as the experiment. This can be done -# by setting the variable envir to the name of the current test. Since -# as mentiond above we will store this name in RUN, below we simply set -# envir to the same value as RUN (which is just EXPT_SUBDIR). Then, for -# this test, the UPP output will be located in the directory -# -# \$PTMP/com/\$NET/\$RUN/\$RUN.\$yyyymmdd/\$hh -# -RUN=\"\${EXPT_SUBDIR}\" -envir=\"\${EXPT_SUBDIR}\"" -# -# Set FIXLAM_NCO_BASEDIR. -# - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - FIXLAM_NCO_BASEDIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - FIXLAM_NCO_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "HERA" ]; then - FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" - elif [ "$MACHINE" = "JET" ]; then - FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" - elif [ "$MACHINE" = "CHEYENNE" ]; then - FIXLAM_NCO_BASEDIR="/needs/to/be/specified" - else - print_err_msg_exit "\ -The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, -orography, and surface climatology \"fixed\" files used in NCO mode are -located has not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - str=${str}" -# -# The base directory in which the pregenerated grid, orography, and surface -# climatology \"fixed\" files used in NCO mode are located. In NCO mode, -# the workflow scripts will create symlinks (in the directory specified -# by FIXLAM) to files in a subdirectory under FIXLAM_NCO_BASDEDIR, where -# the name of the subdirectory is the name of the predefined grid specified -# by PREDEF_GRID_NAME. -# -FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" -# -# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model -# for ICs or LBCs. -# - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - COMINgfs="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "HERA" ]; then - COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "JET" ]; then - COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" - elif [ "$MACHINE" = "CHEYENNE" ]; then - COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" - else - print_err_msg_exit "\ -The directory (COMINgfs) that needs to be specified when running the -workflow in NCO mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or -the GSMGFS as the external model for ICs and/or LBCs has not been specified -for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - str=${str}" -# -# Directory that needs to be specified when running the workflow in NCO -# mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as -# the external model for ICs and/or LBCs. -# -COMINgfs=\"${COMINgfs}\"" - - fi -# -# Set STMP and PTMP. -# - nco_basedir=$( readlink -f "$homerrfs/../../nco_dirs" ) - STMP=${stmp:-"${nco_basedir}/stmp"} - PTMP=${ptmp:-"${nco_basedir}/ptmp"} - - str=${str}" -# -# Directories STMP and PTMP that need to be specified when running the -# workflow in NCO-mode (i.e. RUN_ENVIR set to "nco"). -# -STMP=\"${STMP}\" -PTMP=\"${PTMP}\"" - - fi -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# - if [ ${USE_USER_STAGED_EXTRN_FILES} = "TRUE" ]; then - - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - extrn_mdl_source_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - extrn_mdl_source_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "HERA" ]; then - extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "JET" ]; then - extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "CHEYENNE" ]; then - extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" - elif [ "$MACHINE" = "ORION" ]; then - extrn_mdl_source_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" - else - print_err_msg_exit "\ -The base directory (extrn_mdl_source_basedir) in which the user-staged -external model files should be located has not been specified for this -machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) - fi - elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - fi - - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" -# -# Make sure that the forecast length is evenly divisible by the interval -# between the times at which the lateral boundary conditions will be -# specified. -# - rem=$(( 10#${FCST_LEN_HRS} % 10#${LBC_SPEC_INTVL_HRS} )) - if [ "$rem" -ne "0" ]; then - print_err_msg_exit "\ -The forecast length (FCST_LEN_HRS) must be evenly divisible by the lateral -boundary conditions specification interval (LBC_SPEC_INTVL_HRS): - FCST_LEN_HRS = ${FCST_LEN_HRS} - LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} - rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" - fi - lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) - EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) - fi - elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - fi - - str=${str}" -# -# Locations and names of user-staged external model files for generating -# ICs and LBCs. -# -EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" -EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" -EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" - - fi - -# -#----------------------------------------------------------------------- -# -# Set MET and MET+ paths, if necessary. -# -#----------------------------------------------------------------------- -# - if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then - if [ "$MACHINE" = "HERA" ]; then - met_install_dir="/contrib/met/10.0.0" - metplus_path="/contrib/METplus/METplus-4.0.0" - else - print_err_msg_exit "\ -The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) have not been specified for -this machine (MACHINE): MACHINE= \"${MACHINE}\" " - fi - - str=${str}" -# -# MET and MET+ paths. -# -METPLUS_PATH=\"${metplus_path}\" -MET_INSTALL_DIR=\"${met_install_dir}\"" - - fi - -# -#----------------------------------------------------------------------- -# -# On some machines (e.g. cheyenne), some tasks take many attempts to -# succeed. To make it more convenient to run the WE2E tests on these -# machines without manual intervention, change the number of attempts -# for such tasks on those machines to be more than one. -# -#----------------------------------------------------------------------- -# - add_maxtries="FALSE" - - if [ "$MACHINE" = "HERA" ]; then - add_maxtries="TRUE" - MAXTRIES_MAKE_ICS="2" - MAXTRIES_MAKE_LBCS="2" - MAXTRIES_RUN_POST="2" - elif [ "$MACHINE" = "CHEYENNE" ]; then - add_maxtries="TRUE" - MAXTRIES_MAKE_SFC_CLIMO="3" - MAXTRIES_MAKE_ICS="5" - MAXTRIES_MAKE_LBCS="10" - MAXTRIES_RUN_POST="10" - fi - - if [ "${add_maxtries}" = "TRUE" ]; then - - str=${str}" -# -# Maximum number of attempts at running each task. -# -MAXTRIES_MAKE_GRID=\"${MAXTRIES_MAKE_GRID}\" -MAXTRIES_MAKE_OROG=\"${MAXTRIES_MAKE_OROG}\" -MAXTRIES_MAKE_SFC_CLIMO=\"${MAXTRIES_MAKE_SFC_CLIMO}\" -MAXTRIES_GET_EXTRN_ICS=\"${MAXTRIES_GET_EXTRN_ICS}\" -MAXTRIES_GET_EXTRN_LBCS=\"${MAXTRIES_GET_EXTRN_LBCS}\" -MAXTRIES_MAKE_ICS=\"${MAXTRIES_MAKE_ICS}\" -MAXTRIES_MAKE_LBCS=\"${MAXTRIES_MAKE_LBCS}\" -MAXTRIES_RUN_FCST=\"${MAXTRIES_RUN_FCST}\" -MAXTRIES_RUN_POST=\"${MAXTRIES_RUN_POST}\"" - - fi -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# - printf "%s" "$str" > "${expt_config_fp}" -# -#----------------------------------------------------------------------- -# -# If EXTRN_MDL_SYSBASEDIR_ICS has been specified in the current WE2E -# test's base configuration file, it must be set to one of the following: -# -# 1) The string "set_to_non_default_location_in_testing_script" in order -# to allow this script to set it to a valid location depending on the -# machine and external model (for ICs). -# -# 2) To an existing directory. If it is set to a directory, then this -# script ensures that the directory exists (via the check below). -# -#----------------------------------------------------------------------- -# - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - - if [ "${EXTRN_MDL_SYSBASEDIR_ICS}" = "set_to_non_default_location_in_testing_script" ]; then - - EXTRN_MDL_SYSBASEDIR_ICS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi - - if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - print_err_msg_exit "\ -A non-default location for EXTRN_MDL_SYSBASEDIR_ICS for testing purposes -has not been specified for this machine (MACHINE) and external model for -initial conditions (EXTRN_MDL_NAME_ICS) combination: - MACHINE= \"${MACHINE}\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" - fi - - else - - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - print_err_msg_exit "\ -The non-default location specified by EXTRN_MDL_SYSBASEDIR_ICS does not -exist or is not a directory: - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" - fi - - fi - - set_bash_param "${expt_config_fp}" "EXTRN_MDL_SYSBASEDIR_ICS" "${EXTRN_MDL_SYSBASEDIR_ICS}" - - fi -# -#----------------------------------------------------------------------- -# -# Same as above but for EXTRN_MDL_SYSBASEDIR_LBCS. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - - if [ "${EXTRN_MDL_SYSBASEDIR_LBCS}" = "set_to_non_default_location_in_testing_script" ]; then - - EXTRN_MDL_SYSBASEDIR_LBCS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi - - if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - print_err_msg_exit "\ -A non-default location for EXTRN_MDL_SYSBASEDIR_LBCS for testing purposes -has not been specified for this machine (MACHINE) and external model for -initial conditions (EXTRN_MDL_NAME_LBCS) combination: - MACHINE= \"${MACHINE}\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" - fi - - else - - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - print_err_msg_exit "\ -The non-default location specified by EXTRN_MDL_SYSBASEDIR_LBCS does not -exist or is not a directory: - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" - fi - - fi - - set_bash_param "${expt_config_fp}" "EXTRN_MDL_SYSBASEDIR_LBCS" "${EXTRN_MDL_SYSBASEDIR_LBCS}" - - fi -# -#----------------------------------------------------------------------- -# -# Set the values of those parameters in the experiment configuration file -# that need to be adjusted from their baseline values (as specified in -# the current line of the experiments list file) to obtain the configuration -# file for the current experiment. -# -#----------------------------------------------------------------------- -# - printf "" - for (( j=0; j<${num_mod_vars}; j++ )); do - set_bash_param "${expt_config_fp}" "${modvar_name[$j]}" "${modvar_value[$j]}" - done -# -# Move the current experiment's configuration file into the directory in -# which the experiment generation script expects to find it, and in the -# process rename the file to the name that the experiment generation script -# expects it to have. -# - mv_vrfy -f "${expt_config_fp}" "$ushdir/${EXPT_CONFIG_FN}" -# -#----------------------------------------------------------------------- -# -# Call the experiment/workflow generation script to generate an experi- -# ment directory and rocoto workflow XML for the current experiment. -# -#----------------------------------------------------------------------- -# - $ushdir/generate_FV3LAM_wflow.sh || \ - print_err_msg_exit "\ -Could not generate an experiment/workflow for the test specified by -expt_name: - expt_name = \"${expt_name}\"" - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/tests/testlist.release_public_v1.txt b/tests/testlist.release_public_v1.txt deleted file mode 100644 index 662e3408b4..0000000000 --- a/tests/testlist.release_public_v1.txt +++ /dev/null @@ -1,7 +0,0 @@ -GST_release_public_v1 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha -grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha -grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index d73da5ef27..57a0870580 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -127,8 +127,8 @@ are zero or more options and arguments to pass to that command. # #----------------------------------------------------------------------- # - output=$( "$cmd" "$@" 2>&1 ) - exit_code=$? + local output=$( "$cmd" "$@" 2>&1 ) + local exit_code=$? # #----------------------------------------------------------------------- # @@ -139,7 +139,7 @@ are zero or more options and arguments to pass to that command. #----------------------------------------------------------------------- # if [ -n "$output" ]; then - double_space=" " + local double_space=" " output="${double_space}${output}" output=${output/$'\n'/$'\n'${double_space}} fi @@ -153,9 +153,9 @@ are zero or more options and arguments to pass to that command. # if [ "${caller_name}" = "main" ] || \ [ "${caller_name}" = "script" ]; then - script_or_function="the script" + local script_or_function="the script" else - script_or_function="function \"${caller_name}\"" + local script_or_function="function \"${caller_name}\"" fi if [ ${exit_code} -ne 0 ]; then diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh index bd92c6f3fd..d3060db7d3 100644 --- a/ush/bash_utils/is_element_of.sh +++ b/ush/bash_utils/is_element_of.sh @@ -112,7 +112,7 @@ Use this function in a script as follows: str_to_match="$2" array_name_at="$array_name[@]" - array=("${!array_name_at}") + array=("${!array_name_at:-}") # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 1299a86709..da3742043b 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1110,7 +1110,7 @@ PREEXISTING_DIR_METHOD="delete" #----------------------------------------------------------------------- # # Set VERBOSE. This is a flag that determines whether or not the experiment -# generation and workflow task scripts tend to be print out more informational +# generation and workflow task scripts tend to print out more informational # messages. # #----------------------------------------------------------------------- @@ -1119,38 +1119,65 @@ VERBOSE="TRUE" # #----------------------------------------------------------------------- # -# Set flags (and related directories) that determine whether the grid, -# orography, and/or surface climatology file generation tasks should be -# run. Note that these are all cycle-independent tasks, i.e. if they are -# to be run, they do so only once at the beginning of the workflow before -# any cycles are run. Definitions: +# Set the names of the various rocoto workflow tasks. +# +#----------------------------------------------------------------------- +# +MAKE_GRID_TN="make_grid" +MAKE_OROG_TN="make_orog" +MAKE_SFC_CLIMO_TN="make_sfc_climo" +GET_EXTRN_ICS_TN="get_extrn_ics" +GET_EXTRN_LBCS_TN="get_extrn_lbcs" +MAKE_ICS_TN="make_ics" +MAKE_LBCS_TN="make_lbcs" +RUN_FCST_TN="run_fcst" +RUN_POST_TN="run_post" +GET_OBS="get_obs" +GET_OBS_CCPA_TN="get_obs_ccpa" +GET_OBS_MRMS_TN="get_obs_mrms" +GET_OBS_NDAS_TN="get_obs_ndas" +VX_TN="run_vx" +VX_GRIDSTAT_TN="run_gridstatvx" +VX_GRIDSTAT_REFC_TN="run_gridstatvx_refc" +VX_GRIDSTAT_RETOP_TN="run_gridstatvx_retop" +VX_GRIDSTAT_03h_TN="run_gridstatvx_03h" +VX_GRIDSTAT_06h_TN="run_gridstatvx_06h" +VX_GRIDSTAT_24h_TN="run_gridstatvx_24h" +VX_POINTSTAT_TN="run_pointstatvx" +# +#----------------------------------------------------------------------- +# +# Set flags (and related directories) that determine whether various +# workflow tasks should be run. Note that the MAKE_GRID_TN, MAKE_OROG_TN, +# and MAKE_SFC_CLIMO_TN are all cycle-independent tasks, i.e. if they +# are to be run, they do so only once at the beginning of the workflow +# before any cycles are run. Definitions: # # RUN_TASK_MAKE_GRID: -# Flag that determines whether the grid file generation task is to be run. -# If this is set to "TRUE", the grid generation task is run and new grid -# files are generated. If it is set to "FALSE", then the scripts look -# for pregenerated grid files in the directory specified by GRID_DIR (see -# below). +# Flag that determines whether the MAKE_GRID_TN task is to be run. If +# this is set to "TRUE", the grid generation task is run and new grid +# files are generated. If it is set to "FALSE", then the scripts look +# for pregenerated grid files in the directory specified by GRID_DIR +# (see below). # # GRID_DIR: # The directory in which to look for pregenerated grid files if # RUN_TASK_MAKE_GRID is set to "FALSE". # # RUN_TASK_MAKE_OROG: -# Same as RUN_TASK_MAKE_GRID but for the orography generation task. +# Same as RUN_TASK_MAKE_GRID but for the MAKE_OROG_TN task. # # OROG_DIR: -# Same as GRID_DIR but for the orogrpahy generation task. +# Same as GRID_DIR but for the MAKE_OROG_TN task. # # RUN_TASK_MAKE_SFC_CLIMO: -# Same as RUN_TASK_MAKE_GRID but for the surface climatology generation -# task. +# Same as RUN_TASK_MAKE_GRID but for the MAKE_SFC_CLIMO_TN task. # # SFC_CLIMO_DIR: -# Same as GRID_DIR but for the surface climatology generation task. +# Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # # RUN_TASK_RUN_POST: -# Flag that determines whether the run_post task is to be run +# Flag that determines whether the RUN_POST_TN task is to be run. # # RUN_TASK_VX_GRIDSTAT: # Flag that determines whether the grid-stat verification task is to be @@ -1182,7 +1209,6 @@ RUN_TASK_GET_OBS_NDAS="FALSE" RUN_TASK_VX_GRIDSTAT="FALSE" RUN_TASK_VX_POINTSTAT="FALSE" - # #----------------------------------------------------------------------- # @@ -1367,39 +1393,14 @@ CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING=( \ # #----------------------------------------------------------------------- # -# Set the names of the various workflow tasks. Then, for each task, set -# the parameters to pass to the job scheduler (e.g. slurm) that will submit -# a job for each task to be run. These parameters include the number of -# nodes to use to run the job, the MPI processes per node, the maximum -# walltime to allow for the job to complete, and the maximum number of -# times to attempt to run each task. +# For each workflow task, set the parameters to pass to the job scheduler +# (e.g. slurm) that will submit a job for each task to be run. These +# parameters include the number of nodes to use to run the job, the MPI +# processes per node, the maximum walltime to allow for the job to complete, +# and the maximum number of times to attempt to run each task. # #----------------------------------------------------------------------- # -# Task names. -# -MAKE_GRID_TN="make_grid" -MAKE_OROG_TN="make_orog" -MAKE_SFC_CLIMO_TN="make_sfc_climo" -GET_EXTRN_ICS_TN="get_extrn_ics" -GET_EXTRN_LBCS_TN="get_extrn_lbcs" -MAKE_ICS_TN="make_ics" -MAKE_LBCS_TN="make_lbcs" -RUN_FCST_TN="run_fcst" -RUN_POST_TN="run_post" -GET_OBS="get_obs" -GET_OBS_CCPA_TN="get_obs_ccpa" -GET_OBS_MRMS_TN="get_obs_mrms" -GET_OBS_NDAS_TN="get_obs_ndas" -VX_TN="run_vx" -VX_GRIDSTAT_TN="run_gridstatvx" -VX_GRIDSTAT_REFC_TN="run_gridstatvx_refc" -VX_GRIDSTAT_RETOP_TN="run_gridstatvx_retop" -VX_GRIDSTAT_03h_TN="run_gridstatvx_03h" -VX_GRIDSTAT_06h_TN="run_gridstatvx_06h" -VX_GRIDSTAT_24h_TN="run_gridstatvx_24h" -VX_POINTSTAT_TN="run_pointstatvx" -# # Number of nodes. # NNODES_MAKE_GRID="1" From e05230de9db48a32c525f9557162be7e7dd85e28 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:24:45 -0600 Subject: [PATCH 071/203] Revert "Modifications to allow workflow to run to completion with GNU build on Hera (#526)" (#559) This reverts commit bb28b5fd2174faa7d55aa8b597b79eb265a50e86. --- scripts/exregional_make_ics.sh | 7 --- scripts/exregional_make_lbcs.sh | 7 --- scripts/exregional_run_fcst.sh | 7 --- ush/bash_utils/unload_python.sh | 97 --------------------------------- ush/source_util_funcs.sh | 6 -- 5 files changed, 124 deletions(-) delete mode 100644 ush/bash_utils/unload_python.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 98a82d815e..41b00ac637 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -608,13 +608,6 @@ $settings" # of chgres_cube is nonzero. # A similar thing happens in the forecast task. # -#----------------------------------------------------------------------- -# Deactivate the conda environment to avoid conflicts between the -# environment used by the regional workflow (prepended to your $PATH) -# and that used in the run environment. -#----------------------------------------------------------------------- -unload_python - ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 1ec989c88d..3865f092a1 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -523,13 +523,6 @@ $settings" # of chgres_cube is nonzero. # A similar thing happens in the forecast task. # -#----------------------------------------------------------------------- -# Deactivate the conda environment to avoid conflicts between the -# environment used by the regional workflow (prepended to your $PATH) -# and that used in the run environment. -#----------------------------------------------------------------------- - unload_python - ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 9750dee701..842926891b 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -523,13 +523,6 @@ Call to function to create a diag table file for the current cycle's # #----------------------------------------------------------------------- # -#----------------------------------------------------------------------- -# Deactivate the conda environment to avoid conflicts between the -# environment used by the regional workflow (prepended to your $PATH) -# and that used in the run environment. -#----------------------------------------------------------------------- -unload_python - $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." diff --git a/ush/bash_utils/unload_python.sh b/ush/bash_utils/unload_python.sh deleted file mode 100644 index f5879de0c0..0000000000 --- a/ush/bash_utils/unload_python.sh +++ /dev/null @@ -1,97 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that detects if a python or miniconds3 -# module is loaded, and if so, unloads that module. This may be -# necessary on machines where the loaded python environment isn't -# compatible with environment set for the compiler. -# -#----------------------------------------------------------------------- -# -function unload_python() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 0 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} - -" - - fi -# -#----------------------------------------------------------------------- -# -# If the miniconda or python modules are loaded, unload them -# -#----------------------------------------------------------------------- -# - - modules_to_unload=( python miniconda3 ) - loaded_modules=$(module list 2>&1) - - for module_to_unload in ${modules_to_unload[@]}; do - if [[ "${loaded_modules}" =~ "${module_to_unload}" ]]; then - print_info_msg "\ -Unloading module ${module_to_unload} libraries needed for workflow generation but not for running the workflow... " - module unload ${module_to_unload} - fi - done - - loaded_modules=$(module list 2>&1) - print_info_msg "\ -Loaded modules are: $loaded_modules " - -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index d8b866b69d..bc6a2f0832 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -183,12 +183,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Unload python or miniconda3 if loaded -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/unload_python.sh -# # Source the file containing the function that creates a symlink to a # file (including performing checks). # From 76465bd0d0fae778e7c315907b6659366b70d4d8 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 23 Jul 2021 15:33:45 -0400 Subject: [PATCH 072/203] Remove domains_stack_size from input.yml (#558) --- ush/templates/FV3.input.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 6257232e47..4d7c99b541 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -79,8 +79,6 @@ FV3_HRRR: FV3_GFS_2017_gfdlmp: atmos_model_nml: avg_max_length: 3600.0 - fms_nml: - domains_stack_size: 1800200 fv_core_nml: &gfs_2017_gfdlmp_fv_core agrid_vel_rst: False d4_bg: 0.15 @@ -175,8 +173,6 @@ FV3_GFS_2017_gfdlmp: FV3_GFS_2017_gfdlmp_regional: atmos_model_nml: avg_max_length: 3600.0 - fms_nml: - domains_stack_size: 1800200 fv_core_nml: <<: *gfs_2017_gfdlmp_fv_core k_split: 2 @@ -209,8 +205,6 @@ FV3_GFS_2017_gfdlmp_regional: <<: *gfs_gfdl_cloud_mp FV3_GFS_v15p2: - fms_nml: - domains_stack_size: 1800200 fv_core_nml: &gfs_v15_fv_core agrid_vel_rst: False d2_bg_k1: 0.15 @@ -298,8 +292,6 @@ FV3_GFS_v16: fhouthf: 1 cires_ugwp_nml: launch_level: 27 - fms_nml: - domains_stack_size: 1800200 fv_core_nml: <<: *gfs_v15_fv_core agrid_vel_rst: false @@ -361,8 +353,6 @@ FV3_GFS_v16: surf_map_nml: !!python/none FV3_CPT_v0: - fms_nml: - domains_stack_size: 1800200 fv_core_nml: <<: *gfs_v15_fv_core dnats: 0 From c8e1001d8b14b485ecda46003264e64673e4b234 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Tue, 3 Aug 2021 11:12:31 -0600 Subject: [PATCH 073/203] =?UTF-8?q?develop:=20Verification=20changes=20fro?= =?UTF-8?q?m=20the=20RRFS=5Fbaseline=20branch=20that=20are=20also=20requ?= =?UTF-8?q?=E2=80=A6=20(#565)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modulefiles/tasks/hera/get_obs.local | 4 + scripts/exregional_get_mrms_files.sh | 8 +- ush/config_defaults.sh | 2 +- .../diag_table.FV3_GFS_2017_gfdlmp_regional | 3 + ush/templates/diag_table.FV3_GSD_SAR | 1 + ush/templates/diag_table.FV3_GSD_SAR_v1 | 1 + ush/templates/diag_table.FV3_GSD_v0 | 1 + ush/templates/diag_table.FV3_HRRR | 1 + ush/templates/diag_table.FV3_RRFS_v0 | 1 + ush/templates/diag_table.FV3_RRFS_v1alpha | 1 + ush/templates/diag_table.FV3_RRFS_v1beta | 1 + ush/templates/parm/met/GridDiagConfig | 42 +++++++ .../parm/metplus/APCP_03h.conf_3hCCPA | 108 ------------------ .../parm/metplus/APCP_06h.conf_6hCCPA | 108 ------------------ ush/templates/parm/metplus/GridDiag_REFC.conf | 89 +++++++++++++++ .../parm/metplus/PointStat_conus_sfc.conf | 2 +- .../parm/metplus/PointStat_upper_air.conf | 2 +- ush/templates/parm/metplus/REFC.conf | 10 +- ush/templates/parm/metplus/RETOP.conf | 5 +- 19 files changed, 161 insertions(+), 229 deletions(-) create mode 100755 ush/templates/parm/met/GridDiagConfig delete mode 100644 ush/templates/parm/metplus/APCP_03h.conf_3hCCPA delete mode 100644 ush/templates/parm/metplus/APCP_06h.conf_6hCCPA create mode 100755 ush/templates/parm/metplus/GridDiag_REFC.conf diff --git a/modulefiles/tasks/hera/get_obs.local b/modulefiles/tasks/hera/get_obs.local index 47c1c6f31d..c41c17de83 100644 --- a/modulefiles/tasks/hera/get_obs.local +++ b/modulefiles/tasks/hera/get_obs.local @@ -7,3 +7,7 @@ module purge module load hpss +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV pygraf diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index 0b14f0e605..b21e27f2cb 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -2,6 +2,7 @@ # This script pulls MRMS data from the NOAA HPSS # Top-level MRMS directory +set -x mrms_dir=${OBS_DIR}/.. if [[ ! -d "$mrms_dir" ]]; then mkdir -p $mrms_dir @@ -63,8 +64,10 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do # Set field of interest from the MRMS products, including name and level information. if [ "${field}" = "REFC" ]; then - field_base_name="MergedReflectivityQComposite" - level="_00.00_" + #field_base_name="MergedReflectivityQComposite" + #level="_00.00_" + field_base_name="MergedReflectivityQCComposite" + level="_00.50_" elif [ "${field}" = "RETOP" ]; then field_base_name="EchoTop" level="_18_00.50_" @@ -110,6 +113,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do if [[ ${Status} != 0 ]]; then echo "WARNING: Bad return status (${Status}) for date \"${CurDate}\". Did you forget to run \"module load hpss\"?" echo "WARNING: ${TarCommand}" + exit ${Status} else if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then mkdir -p $mrms_proc/${vyyyymmdd} diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index da3742043b..037a171c22 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -469,7 +469,7 @@ WRITE_DOPOST="FALSE" # use of predetermined directory structure and file names. Therefore, if # the MRMS files are user provided, they need to follow the anticipated # naming structure: -# {YYYYMMDD}/MergedReflectivityQComposite_00.00_{YYYYMMDD}-{HH}{mm}{SS}.grib2, +# {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, # where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD # the 2-digit valid day of the month, HH the 2-digit valid hour of the # day, mm the 2-digit valid minutes of the hour, and SS is the two-digit diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional index d44e7efbf4..f94509ac7d 100644 --- a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional +++ b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional @@ -308,6 +308,9 @@ # Reflectivity from GFDL microphysics "gfs_phys", "refl_10cm", "refl_10cm", "fv3_history2d", "all", .false., "none", 2 +#Cloud fraction +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 + #Max/Min must be kept in separate files. Time is controlled by diag_table # "dynamics", "uh25", "MXUPHL2_5km", "maxmin2D", "all", max, "none", 2 # "dynamics", "uh25", "MNUPHL2_5km", "maxmin2D", "all", min, "none", 2 diff --git a/ush/templates/diag_table.FV3_GSD_SAR b/ush/templates/diag_table.FV3_GSD_SAR index 1482af2aa3..0d2849c346 100644 --- a/ush/templates/diag_table.FV3_GSD_SAR +++ b/ush/templates/diag_table.FV3_GSD_SAR @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GSD_SAR_v1 b/ush/templates/diag_table.FV3_GSD_SAR_v1 index 99034ff0a9..99b7652910 100644 --- a/ush/templates/diag_table.FV3_GSD_SAR_v1 +++ b/ush/templates/diag_table.FV3_GSD_SAR_v1 @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GSD_v0 b/ush/templates/diag_table.FV3_GSD_v0 index 1482af2aa3..0d2849c346 100644 --- a/ush/templates/diag_table.FV3_GSD_v0 +++ b/ush/templates/diag_table.FV3_GSD_v0 @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_HRRR b/ush/templates/diag_table.FV3_HRRR index 1482af2aa3..0d2849c346 100644 --- a/ush/templates/diag_table.FV3_HRRR +++ b/ush/templates/diag_table.FV3_HRRR @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v0 b/ush/templates/diag_table.FV3_RRFS_v0 index 99034ff0a9..99b7652910 100644 --- a/ush/templates/diag_table.FV3_RRFS_v0 +++ b/ush/templates/diag_table.FV3_RRFS_v0 @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v1alpha b/ush/templates/diag_table.FV3_RRFS_v1alpha index 99034ff0a9..99b7652910 100644 --- a/ush/templates/diag_table.FV3_RRFS_v1alpha +++ b/ush/templates/diag_table.FV3_RRFS_v1alpha @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v1beta b/ush/templates/diag_table.FV3_RRFS_v1beta index 99034ff0a9..99b7652910 100644 --- a/ush/templates/diag_table.FV3_RRFS_v1beta +++ b/ush/templates/diag_table.FV3_RRFS_v1beta @@ -118,6 +118,7 @@ "gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 "gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 #"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 diff --git a/ush/templates/parm/met/GridDiagConfig b/ush/templates/parm/met/GridDiagConfig new file mode 100755 index 0000000000..baa6c8d5bb --- /dev/null +++ b/ush/templates/parm/met/GridDiagConfig @@ -0,0 +1,42 @@ +///////////////////////////////////////////////////////////////////////////////// +// +// Grid-Diag configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Data description +// +${METPLUS_DESC} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Output grid +// +${METPLUS_REGRID_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +${METPLUS_CENSOR_THRESH} +${METPLUS_CENSOR_VAL} + +// +// Data fields +// +${METPLUS_DATA_DICT} + +${METPLUS_MASK_DICT} + +${METPLUS_MET_CONFIG_OVERRIDES} + +//////////////////////////////////////////////////////////////////////////////// + +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA b/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA deleted file mode 100644 index 3235ff6316..0000000000 --- a/ush/templates/parm/metplus/APCP_03h.conf_3hCCPA +++ /dev/null @@ -1,108 +0,0 @@ -# Grid to Grid Precipitation - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd -OUTPUT_BASE = {ENV[EXPTDIR]} -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine -FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} - -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -[filename_templates] -# format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h -FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} - -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.03h.hrap.conus.gb2 - -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = PcpCombine, GridStat - -# Run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = True -OBS_PCP_COMBINE_RUN = False - -# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) -FCST_PCP_COMBINE_METHOD = ADD - -FCST_PCP_COMBINE_CONSTANT_INIT = True - -# Accumulation interval available in forecast data -FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 - -# If 'bucket' output already exists, skip the PcpCombine step for the data -PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True - -# Forecast data description variables -FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB -FCST_IS_PROB = false - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr({ENV[ACCUM]},{ENV[LAST_HR]},{ENV[ACCUM]}) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A03 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A03 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - diff --git a/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA b/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA deleted file mode 100644 index d3d1a9f205..0000000000 --- a/ush/templates/parm/metplus/APCP_06h.conf_6hCCPA +++ /dev/null @@ -1,108 +0,0 @@ -# Grid to Grid Precipitation - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd -OUTPUT_BASE = {ENV[EXPTDIR]} -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine -FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} - -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -[filename_templates] -# format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h -FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} - -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.06h.hrap.conus.gb2 - -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = PcpCombine, GridStat - -# Run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = True -OBS_PCP_COMBINE_RUN = False - -# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) -FCST_PCP_COMBINE_METHOD = ADD - -FCST_PCP_COMBINE_CONSTANT_INIT = True - -# Accumulation interval available in forecast data -FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 - -# If 'bucket' output already exists, skip the PcpCombine step for the data -PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True - -# Forecast data description variables -FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB -FCST_IS_PROB = false - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr({ENV[ACCUM]},{ENV[LAST_HR]},{ENV[ACCUM]}) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A06 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A06 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - diff --git a/ush/templates/parm/metplus/GridDiag_REFC.conf b/ush/templates/parm/metplus/GridDiag_REFC.conf new file mode 100755 index 0000000000..d148800fb5 --- /dev/null +++ b/ush/templates/parm/metplus/GridDiag_REFC.conf @@ -0,0 +1,89 @@ +# +# CONFIGURATION +# +[config] + +INPUT_BASE={ENV[INPUT_BASE]} +OUTPUT_BASE={ENV[OUTPUT_BASE]} + +# 'Tasks' to be run +PROCESS_LIST = GridDiag + +GRID_DIAG_RUNTIME_FREQ = RUN_ONCE +#other options include: RUN_ONCE_PER_INIT_OR_VALID, RUN_ONCE_PER_LEAD, RUN_ONCE_FOR_EACH + +LOOP_BY = INIT + +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG={ENV[INIT_BEG]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END={ENV[INIT_END]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = {ENV[INIT_INC]} + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = begin_end_incr({ENV[FHR_FIRST]},{ENV[FHR_LAST]},{ENV[FHR_INC]}) + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +#LOOP_ORDER = times +LOOP_ORDER = processes +# must be processes when uisng GRID_DIAG_RUNTIME_FREQ + +LOG_GRID_DIAG_VERBOSITY = 2 + +GRID_DIAG_DESCRIPTION = RRFS +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} + +# Configuration file +GRID_DIAG_CONFIG_FILE = {CONFIG_DIR}/GridDiagConfig + +BOTH_VAR1_NAME = REFC +BOTH_VAR1_LEVELS = L0 +BOTH_VAR1_OPTIONS = cnt_thresh = [ >15 ]; n_bins = 18; range = [-20, 70]; + +BOTH_VAR2_NAME = MergedReflectivityQCComposite +BOTH_VAR2_LEVELS = Z500 +BOTH_VAR2_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; n_bins = 18; range = [-20, 70]; + +# The following variables set values in the MET +# configuration file used by this example +# Leaving these values commented will use the value +# found in the default MET configuration file +GRID_DIAG_REGRID_TO_GRID = FCST +#GRID_DIAG_REGRID_METHOD = BUDGET +#GRID_DIAG_REGRID_WIDTH = 2 +#GRID_DIAG_REGRID_VLD_THRESH = 0.5 +#GRID_DIAG_REGRID_SHAPE = SQUARE + +# DIRECTORIES +# +[dir] + +GRID_DIAG_INPUT_DIR = {INPUT_BASE} + +GRID_DIAG_OUTPUT_DIR = {OUTPUT_BASE} + +[filename_templates] + +GRID_DIAG_INPUT_TEMPLATE = expt_dirs/RRFS_baseline_summer/{init?fmt=%Y%m%d%H}/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, obs_data/mrms/proc/{valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{ENV[INIT_BEG]}-{ENV[INIT_END]}_f{ENV[FHR_FIRST]}-{ENV[FHR_LAST]}.nc +#when using RUN_ONCE +#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{init?fmt=%Y%m%d%H}.nc when using RUN_ONCE_PER_INIT_OR_VALID +#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{lead?fmt=%3H}.nc +# when using RUN_ONCE_PER_LEAD +#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.nc when using RUN_ONCE_FOR_EACH + +GRID_DIAG_VERIFICATION_MASK_TEMPLATE = /scratch2/BMC/fv3lam/RRFS_baseline/static/MET/masks/CONUS.nc diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 69f315fe31..27e08d0589 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -108,7 +108,7 @@ POINT_STAT_GRID = # environment variable that is too long, resulting in an error. For long # lists of poly masking files (i.e. all the mask files in the NCEP_mask # directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index bed01d8e6c..d0be4bebf2 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -108,7 +108,7 @@ POINT_STAT_GRID = # environment variable that is too long, resulting in an error. For long # lists of poly masking files (i.e. all the mask files in the NCEP_mask # directory), define these in the MET point_stat configuration file. -POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf index db4864043a..94200119a7 100644 --- a/ush/templates/parm/metplus/REFC.conf +++ b/ush/templates/parm/metplus/REFC.conf @@ -17,8 +17,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 # ANLYS -#OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/MergedReflectivityQComposite_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQComposite_00.00_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat @@ -75,12 +74,13 @@ GRID_STAT_REGRID_TO_GRID = FCST # Forecast/Observation variable Information FCST_VAR1_NAME = REFC FCST_VAR1_LEVELS = L0 +FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 -OBS_VAR1_NAME = MergedReflectivityQComposite -OBS_VAR1_LEVELS = L0 -OBS_VAR1_OPTIONS = censor_thresh = eq-999; censor_val = -9999; +OBS_VAR1_NAME = MergedReflectivityQCComposite +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 diff --git a/ush/templates/parm/metplus/RETOP.conf b/ush/templates/parm/metplus/RETOP.conf index d95289faf2..68b37d2fce 100644 --- a/ush/templates/parm/metplus/RETOP.conf +++ b/ush/templates/parm/metplus/RETOP.conf @@ -74,14 +74,13 @@ GRID_STAT_REGRID_TO_GRID = FCST # Forecast/Observation variable Information FCST_VAR1_NAME = RETOP FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); +FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; convert(x) = M_to_KFT(x); BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 OBS_VAR1_NAME = EchoTop18 OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = eq-999; censor_val = -9999; convert(x) = KM_to_KFT(x); - +OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 From f9db719526308b66841400f0759ef45cae804ccb Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 3 Aug 2021 19:32:09 -0400 Subject: [PATCH 074/203] Remove ncores_per_node from model_configure and set PPN_RUN_FCST in setup.sh (#563) * remove dummy variables and rearrange variables * Add a defalut value to ppn_run_fcst * Remove conditional statement --- scripts/exregional_run_fcst.sh | 1 - ush/config_defaults.sh | 9 ++------- ush/create_model_configure_file.sh | 4 +--- ush/generate_FV3LAM_wflow.sh | 2 +- ush/setup.sh | 9 +++++++++ ush/templates/model_configure | 1 - 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 842926891b..9fa3ff18b0 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -490,7 +490,6 @@ fi # create_model_configure_file \ cdate="$cdate" \ - nthreads=${OMP_NUM_THREADS} \ run_dir="${run_dir}" \ sub_hourly_post="${SUB_HOURLY_POST}" \ dt_subhourly_post_mnts="${DT_SUBHOURLY_POST_MNTS}" \ diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 037a171c22..151f78df07 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1427,7 +1427,7 @@ PPN_GET_EXTRN_ICS="1" PPN_GET_EXTRN_LBCS="1" PPN_MAKE_ICS="12" PPN_MAKE_LBCS="12" -PPN_RUN_FCST="24" # This may have to be changed depending on the number of threads used. +PPN_RUN_FCST="" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.sh PPN_RUN_POST="24" PPN_GET_OBS_CCPA="1" PPN_GET_OBS_MRMS="1" @@ -1672,9 +1672,6 @@ COMPILER="intel" # Controls the size of the stack for threads created by the OpenMP # implementation. # -# CPUS_PER_TASK_RUN_FCST: -# Sets the number of MPI tasks per CPU for the RUN_FCST task. -# # Note that settings for the make_grid and make_orog tasks are not # included below as they do not use parallelized code. # @@ -1697,11 +1694,9 @@ OMP_NUM_THREADS_MAKE_LBCS="1" OMP_STACKSIZE_MAKE_LBCS="1024m" KMP_AFFINITY_RUN_FCST="scatter" -OMP_NUM_THREADS_RUN_FCST="4" +OMP_NUM_THREADS_RUN_FCST="2" # atmos_nthreads in model_configure OMP_STACKSIZE_RUN_FCST="1024m" -CPUS_PER_TASK_RUN_FCST="2" - KMP_AFFINITY_RUN_POST="scatter" OMP_NUM_THREADS_RUN_POST="1" OMP_STACKSIZE_RUN_POST="1024m" diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index 1be54516ea..23602932bb 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -48,7 +48,6 @@ function create_model_configure_file() { local valid_args=( cdate \ run_dir \ -nthreads \ sub_hourly_post \ dt_subhourly_post_mnts \ dt_atmos \ @@ -122,8 +121,7 @@ run directory (run_dir): 'nhours_fcst': ${FCST_LEN_HRS} 'dt_atmos': ${DT_ATMOS} 'cpl': ${dot_cpl_dot} - 'atmos_nthreads': ${nthreads:-1} - 'ncores_per_node': ${NCORES_PER_NODE} + 'atmos_nthreads': ${OMP_NUM_THREADS_RUN_FCST} 'restart_interval': ${RESTART_INTERVAL} 'write_dopost': ${dot_write_dopost} 'quilting': ${dot_quilting_dot} diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 8f4bbdf3ca..07aee40365 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -218,7 +218,7 @@ settings="\ # Number of cores used for a task # 'ncores_run_fcst': ${PE_MEMBER01} - 'native_run_fcst': --cpus-per-task ${CPUS_PER_TASK_RUN_FCST} --exclusive + 'native_run_fcst': --cpus-per-task ${OMP_NUM_THREADS_RUN_FCST} --exclusive # # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. diff --git a/ush/setup.sh b/ush/setup.sh index 06c8289242..2ab135eedf 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -579,6 +579,15 @@ esac # #----------------------------------------------------------------------- # +# Calculate PPN_RUN_FCST from NCORES_PER_NODE and OMP_NUM_THREADS_RUN_FCST +# +#----------------------------------------------------------------------- +# +PPN_RUN_FCST_OPT="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" +PPN_RUN_FCST=${PPN_RUN_FCST:-${PPN_RUN_FCST_OPT}} +# +#----------------------------------------------------------------------- +# # Make sure that the job scheduler set above is valid. # #----------------------------------------------------------------------- diff --git a/ush/templates/model_configure b/ush/templates/model_configure index bdd812b750..f26b9dd15c 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -15,7 +15,6 @@ calendar: 'julian' memuse_verbose: .false. atmos_nthreads: {{ atmos_nthreads }} use_hyper_thread: .false. -ncores_per_node: {{ ncores_per_node }} debug_affinity: .true. restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. From 2152d8cdfeb1b3431e3a23d8912e2354efeb8e13 Mon Sep 17 00:00:00 2001 From: Siwei He <43216838+siwei-noaa@users.noreply.github.com> Date: Tue, 3 Aug 2021 17:47:48 -0600 Subject: [PATCH 075/203] Bug fixes to WE2E test "MET_verification" (#564) --- tests/WE2E/run_WE2E_tests.sh | 12 +++++++++--- .../wflow_features/config.MET_verification.sh | 11 +++++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 27a23aecb7..8a81dce9ad 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -1022,10 +1022,13 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" if [ "$MACHINE" = "HERA" ]; then met_install_dir="/contrib/met/10.0.0" metplus_path="/contrib/METplus/METplus-4.0.0" + ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ccpa/proc" + mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/mrms/proc" + ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ndas/proc" else print_err_msg_exit "\ -The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) have not -been specified for this machine (MACHINE): +The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) or the observation directories +(CCPA_OBS_DIR, MRMS_OBS_DIR, NDAS_OBS_DIR) have not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi @@ -1034,7 +1037,10 @@ been specified for this machine (MACHINE): # MET and MET+ paths. # METPLUS_PATH=\"${metplus_path}\" -MET_INSTALL_DIR=\"${met_install_dir}\"" +MET_INSTALL_DIR=\"${met_install_dir}\" +CCPA_OBS_DIR=\"${ccpa_obs_dir}\" +MRMS_OBS_DIR=\"${mrms_obs_dir}\" +NDAS_OBS_DIR=\"${ndas_obs_dir}\"" fi # diff --git a/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh index 0af0eb203e..8d7cc8d4c5 100644 --- a/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh +++ b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh @@ -12,15 +12,18 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" +USE_USER_STAGED_EXTRN_FILES="TRUE" + EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FV3GFS_FILE_FMT_LBCS="grib2" -DATE_FIRST_CYCL="20190415" -DATE_LAST_CYCL="20190415" +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" CYCL_HRS=( "00" ) -FCST_LEN_HRS="36" +FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="6" MODEL="FV3_GFS_v15p2_CONUS_25km" From 94782765f51ac334d6afa3cdaaa1065f10d7ac75 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 5 Aug 2021 09:42:02 -0600 Subject: [PATCH 076/203] Add the pazi parameter option to the creation of the regional_esg_grid namelist; set pazi=-13.0 for the RRFS_NA_3km domain (#567) --- scripts/exregional_make_grid.sh | 1 + ush/config_defaults.sh | 4 ++++ ush/set_gridparams_ESGgrid.sh | 3 +++ ush/set_predef_grid_params.sh | 36 +++++++++++++++++++++++++++++++++ ush/setup.sh | 3 +++ ush/templates/regional_grid.nml | 5 +++++ 6 files changed, 52 insertions(+) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index b68259310e..5d9bddaf66 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -357,6 +357,7 @@ generation executable (exec_fp): 'dely': ${DEL_ANGLE_Y_SG}, 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO}, 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO}, + 'pazi': ${PAZI}, } " # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 151f78df07..64baa7a8a0 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -898,6 +898,9 @@ GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="" # the regional grid before shaving the halo down to the width(s) expected # by the forecast model. # +# ESGgrid_PAZI: +# The rotational parameter for the ESG grid (in degrees). +# # In order to generate grid files containing halos that are 3-cell and # 4-cell wide and orography files with halos that are 0-cell and 3-cell # wide (all of which are required as inputs to the forecast model), the @@ -952,6 +955,7 @@ ESGgrid_DELY="" ESGgrid_NX="" ESGgrid_NY="" ESGgrid_WIDE_HALO_WIDTH="" +ESGgrid_PAZI="" # #----------------------------------------------------------------------- # diff --git a/ush/set_gridparams_ESGgrid.sh b/ush/set_gridparams_ESGgrid.sh index 3c0fa0f92a..91ec8b83df 100644 --- a/ush/set_gridparams_ESGgrid.sh +++ b/ush/set_gridparams_ESGgrid.sh @@ -55,10 +55,12 @@ function set_gridparams_ESGgrid() { "halo_width" \ "delx" \ "dely" \ +"pazi" \ "output_varname_lon_ctr" \ "output_varname_lat_ctr" \ "output_varname_nx" \ "output_varname_ny" \ +"output_varname_pazi" \ "output_varname_halo_width" \ "output_varname_stretch_factor" \ "output_varname_del_angle_x_sg" \ @@ -153,6 +155,7 @@ function set_gridparams_ESGgrid() { eval ${output_varname_ny}="${ny}" eval ${output_varname_halo_width}="${halo_width}" eval ${output_varname_stretch_factor}="${stretch_factor}" + eval ${output_varname_pazi}="${pazi}" eval ${output_varname_del_angle_x_sg}="${del_angle_x_sg}" eval ${output_varname_del_angle_y_sg}="${del_angle_y_sg}" eval ${output_varname_neg_nx_of_dom_with_wide_halo}="${neg_nx_of_dom_with_wide_halo}" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 2084617ecf..698a7e2a8a 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -112,6 +112,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="202" ESGgrid_NY="116" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -156,6 +158,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="396" ESGgrid_NY="232" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-45}" @@ -200,6 +204,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="1748" ESGgrid_NY="1038" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -244,6 +250,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="840" ESGgrid_NY="600" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -291,6 +299,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="320" ESGgrid_NY="240" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" @@ -409,6 +419,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="1380" ESGgrid_NY="1020" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" @@ -638,6 +650,9 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="1344" # Supergrid value 2704 ESGgrid_NY="1152" # Supergrid value 2320 +# Rotation of the ESG grid in degrees. + ESGgrid_PAZI="0.0" + # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo ESGgrid_WIDE_HALO_WIDTH="6" @@ -713,6 +728,9 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="432" # Supergrid value 880 ESGgrid_NY="360" # Supergrid value 736 +# Rotation of the ESG grid in degrees. + ESGgrid_PAZI="0.0" + # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo ESGgrid_WIDE_HALO_WIDTH="6" @@ -788,6 +806,9 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="576" # Supergrid value 1168 ESGgrid_NY="432" # Supergrid value 880 +# Rotation of the ESG grid in degrees. + ESGgrid_PAZI="0.0" + # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo ESGgrid_WIDE_HALO_WIDTH="6" @@ -863,6 +884,9 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="432" # Supergrid value 880 ESGgrid_NY="360" # Supergrid value 736 +# Rotation of the ESG grid in degrees. + ESGgrid_PAZI="0.0" + # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo ESGgrid_WIDE_HALO_WIDTH="6" @@ -930,6 +954,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="345" ESGgrid_NY="230" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-300}" @@ -972,6 +998,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="665" ESGgrid_NY="444" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-180}" @@ -1014,6 +1042,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="2880" ESGgrid_NY="1920" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -1056,6 +1086,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="74" ESGgrid_NY="51" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-600}" @@ -1144,6 +1176,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX="960" ESGgrid_NY="960" + ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-50}" @@ -1187,6 +1221,8 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NX=3640 ESGgrid_NY=2520 + ESGgrid_PAZI="-13.0" + ESGgrid_WIDE_HALO_WIDTH=6 DT_ATMOS="${DT_ATMOS:-36}" diff --git a/ush/setup.sh b/ush/setup.sh index 2ab135eedf..c147f25d49 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2042,6 +2042,7 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then lat_ctr="${ESGgrid_LAT_CTR}" \ nx="${ESGgrid_NX}" \ ny="${ESGgrid_NY}" \ + pazi="${ESGgrid_PAZI}" \ halo_width="${ESGgrid_WIDE_HALO_WIDTH}" \ delx="${ESGgrid_DELX}" \ dely="${ESGgrid_DELY}" \ @@ -2049,6 +2050,7 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then output_varname_lat_ctr="LAT_CTR" \ output_varname_nx="NX" \ output_varname_ny="NY" \ + output_varname_pazi="PAZI" \ output_varname_halo_width="NHW" \ output_varname_stretch_factor="STRETCH_FAC" \ output_varname_del_angle_x_sg="DEL_ANGLE_X_SG" \ @@ -2777,6 +2779,7 @@ NX="${NX}" NY="${NY}" NHW="${NHW}" STRETCH_FAC="${STRETCH_FAC}" +PAZI="${PAZI}" RES_IN_FIXLAM_FILENAMES="${RES_IN_FIXLAM_FILENAMES}" # diff --git a/ush/templates/regional_grid.nml b/ush/templates/regional_grid.nml index befc4eb1d1..fa175c6d57 100644 --- a/ush/templates/regional_grid.nml +++ b/ush/templates/regional_grid.nml @@ -73,6 +73,10 @@ ! -- ! Analogous to lx but in the y direction. ! +! pazi: +! ---- +! Rotation angle for the ESG grid in degrees. +! !*********************************************************************** ! @@ -83,4 +87,5 @@ dely = lx = ly = + pazi = / From 02eb626b8d0524cc32e95faedd693fea3d2a0d92 Mon Sep 17 00:00:00 2001 From: Jili Dong Date: Mon, 9 Aug 2021 15:44:59 -0400 Subject: [PATCH 077/203] fix filter_topo failure when GWD orog is on (#569) --- scripts/exregional_make_orog.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 4930295e7b..ed7d43a0c0 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -325,11 +325,10 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then tmp_dir="${OROG_DIR}/temp_orog_data" mkdir_vrfy -p ${tmp_dir} cd_vrfy ${tmp_dir} - - mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" - mosaic_fp="$FIXLAM/${mosaic_fn}" - grid_fn=$( get_charvar_from_netcdf "${mosaic_fp}" "gridfiles" ) - grid_fp="${FIXLAM}/${grid_fn}" + mosaic_fn_gwd="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" + mosaic_fp_gwd="$FIXLAM/${mosaic_fn_gwd}" + grid_fn_gwd=$( get_charvar_from_netcdf "${mosaic_fp_gwd}" "gridfiles" ) + grid_fp_gwd="${FIXLAM}/${grid_fn_gwd}" ls_fn="geo_em.d01.lat-lon.2.5m.HGT_M.nc" ss_fn="HGT.Beljaars_filtered.lat-lon.30s_res.nc" if [ "${MACHINE}" = "WCOSS_CRAY" ]; then @@ -337,7 +336,7 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then else relative_or_null="--relative" fi - ln_vrfy -fs ${relative_or_null} "${grid_fp}" "${tmp_dir}/${grid_fn}" + ln_vrfy -fs ${relative_or_null} "${grid_fp_gwd}" "${tmp_dir}/${grid_fn_gwd}" ln_vrfy -fs ${relative_or_null} "${FIXam}/${ls_fn}" "${tmp_dir}/${ls_fn}" ln_vrfy -fs ${relative_or_null} "${FIXam}/${ss_fn}" "${tmp_dir}/${ss_fn}" From 916a3c81c27a1fe4c658bae820a9f3544caf39f3 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:34:39 -0400 Subject: [PATCH 078/203] Set up the parameters for MET on WCOSS in run_WE2E_tests.sh (#574) * Update WE2E script for wcoss * Modify path to external data on hera --- tests/WE2E/run_WE2E_tests.sh | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 8a81dce9ad..143be80849 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -950,8 +950,13 @@ external model files should be located has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi - + EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" + + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" + fi + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then @@ -967,6 +972,11 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" + + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" + fi + # # Make sure that the forecast length is evenly divisible by the interval # between the times at which the lateral boundary conditions will be @@ -1019,12 +1029,27 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then - if [ "$MACHINE" = "HERA" ]; then + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + met_install_dir="/gpfs/hps/nco/ops/nwprod/met.v9.1.3" + metplus_path="/gpfs/hps/nco/ops/nwprod/metplus.v3.1.1/METplus-3.1.1" + ccpa_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" + mrms_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" + ndas_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" + met_bin_exec="exec" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + met_install_dir="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" + metplus_path="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" + ccpa_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc" + mrms_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" + ndas_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" + met_bin_exec="exec" + elif [ "$MACHINE" = "HERA" ]; then met_install_dir="/contrib/met/10.0.0" metplus_path="/contrib/METplus/METplus-4.0.0" ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ccpa/proc" mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/mrms/proc" ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ndas/proc" + met_bin_exec="bin" else print_err_msg_exit "\ The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) or the observation directories @@ -1040,7 +1065,8 @@ METPLUS_PATH=\"${metplus_path}\" MET_INSTALL_DIR=\"${met_install_dir}\" CCPA_OBS_DIR=\"${ccpa_obs_dir}\" MRMS_OBS_DIR=\"${mrms_obs_dir}\" -NDAS_OBS_DIR=\"${ndas_obs_dir}\"" +NDAS_OBS_DIR=\"${ndas_obs_dir}\" +MET_BIN_EXEC=\"${met_bin_exec}\"" fi # From c1dddf76b193d97ca0d830bbdbee9b3b092cd208 Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Thu, 26 Aug 2021 10:25:07 -0600 Subject: [PATCH 079/203] Move mrms_pull_topofhour.py from scripts/ to ush/ [develop] (#571) * Moved mrms_pull_topofhour.py to ush/ rather than scripts/ * Updates to get the mrms_pull_topofhour.py script to run with the proper environment loaded. Fixed a tab issue in the py script as well. * Move "set +u" to the place it is absolutely needed, add a "set -u" at the end to restore the unset variable flag Co-authored-by: Michael Kavulich, Jr --- scripts/exregional_get_mrms_files.sh | 4 +++- ush/config_defaults.sh | 2 +- ush/load_modules_run_task.sh | 3 ++- {scripts => ush}/mrms_pull_topofhour.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) rename {scripts => ush}/mrms_pull_topofhour.py (98%) diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index b21e27f2cb..ff09c55462 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -1,5 +1,7 @@ #!/bin/sh +. ${GLOBAL_VAR_DEFNS_FP} + # This script pulls MRMS data from the NOAA HPSS # Top-level MRMS directory set -x @@ -122,7 +124,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do hour=0 while [[ ${hour} -le 23 ]]; do echo "hour=${hour}" - python ${SCRIPTSDIR}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} ${field_base_name} ${level} + python ${USHDIR}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} ${field_base_name} ${level} hour=$((${hour} + 1)) # hourly increment done fi diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 64baa7a8a0..823cf12e89 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -483,7 +483,7 @@ WRITE_DOPOST="FALSE" # HPSS has an example of the expected file naming structure: # regional_workflow/scripts/exregional_get_mrms_files.sh. This script # calls the script used to identify the MRMS file closest to the valid -# time: regional_workflow/scripts/mrms_pull_topofhour.py. +# time: regional_workflow/ush/mrms_pull_topofhour.py. # # NDAS_OBS_DIR: # User-specified location of top-level directory where NDAS prepbufr diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 862bf9338c..8b91ef0cfe 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -328,10 +328,11 @@ ules_dir) for the specified task (task_name) failed: # now. if [ -n "${SRW_ENV:-}" ] ; then + set +u conda activate ${SRW_ENV} + set -u fi - # #----------------------------------------------------------------------- # diff --git a/scripts/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py similarity index 98% rename from scripts/mrms_pull_topofhour.py rename to ush/mrms_pull_topofhour.py index 36d644be38..43bc706193 100644 --- a/scripts/mrms_pull_topofhour.py +++ b/ush/mrms_pull_topofhour.py @@ -73,7 +73,7 @@ print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) - os.system('gunzip '+VALID_DIR+'/'+filename2) + os.system('gunzip '+VALID_DIR+'/'+filename2) elif valid.strftime('%Y%m%d') >= '20200304': print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) From 367abb5cec8bfc33a337ca6fbbe66c338e5c7f7a Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 26 Aug 2021 14:41:21 -0600 Subject: [PATCH 080/203] Fixes to the "pazi" parameter to work with both GFDL and ESG grids. (#583) --- ush/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.sh b/ush/setup.sh index c147f25d49..1794ab7074 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2779,7 +2779,6 @@ NX="${NX}" NY="${NY}" NHW="${NHW}" STRETCH_FAC="${STRETCH_FAC}" -PAZI="${PAZI}" RES_IN_FIXLAM_FILENAMES="${RES_IN_FIXLAM_FILENAMES}" # @@ -2842,6 +2841,7 @@ DEL_ANGLE_X_SG="${DEL_ANGLE_X_SG}" DEL_ANGLE_Y_SG="${DEL_ANGLE_Y_SG}" NEG_NX_OF_DOM_WITH_WIDE_HALO="${NEG_NX_OF_DOM_WITH_WIDE_HALO}" NEG_NY_OF_DOM_WITH_WIDE_HALO="${NEG_NY_OF_DOM_WITH_WIDE_HALO}" +PAZI="${PAZI}" EOM } || print_err_msg_exit "\ Heredoc (cat) command to append grid parameters to variable definitions From 9031286c113f9832eaffa75abbd8ede3a65c83c6 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Thu, 26 Aug 2021 18:54:18 -0600 Subject: [PATCH 081/203] Modify post files names to match what is used in latest EMC_post (#581) * Modify post files names. * Change post executable name in ex-script. --- scripts/exregional_run_fcst.sh | 2 +- scripts/exregional_run_post.sh | 6 +++--- ush/Python/plot_allvars.py | 2 +- ush/Python/plot_allvars_diff.py | 4 ++-- ush/templates/parm/metplus/APCP_01h.conf | 2 +- ush/templates/parm/metplus/APCP_03h.conf | 4 ++-- ush/templates/parm/metplus/APCP_06h.conf | 4 ++-- ush/templates/parm/metplus/APCP_24h.conf | 4 ++-- ush/templates/parm/metplus/GridDiag_REFC.conf | 2 +- ush/templates/parm/metplus/PointStat_conus_sfc.conf | 2 +- ush/templates/parm/metplus/PointStat_upper_air.conf | 2 +- ush/templates/parm/metplus/REFC.conf | 2 +- ush/templates/parm/metplus/RETOP.conf | 2 +- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 9fa3ff18b0..e8a7402bce 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -567,7 +567,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then basetime=$( date --date "$yyyymmdd $hh" +%y%j%H%M ) symlink_suffix="_${basetime}f${fhr}${post_mn}" - fids=( "bgdawp" "bgrd3d" ) + fids=( "prslev" "natlev" ) for fid in "${fids[@]}"; do FID="${fid^^}" post_orig_fn="${FID}.${post_fn_suffix}" diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 87d454813b..b722784330 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -276,7 +276,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." -${APRUN} ${EXECDIR}/ncep_post < itag || print_err_msg_exit "\ +${APRUN} ${EXECDIR}/upp.x < itag || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." # @@ -325,13 +325,13 @@ post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${tmmark}.grib2" # # For convenience, change location to postprd_dir (where the final output # from UPP will be located). Then loop through the two files that UPP -# generates (i.e. "...bgdawp..." and "...bgrd3d..." files) and move, +# generates (i.e. "...prslev..." and "...natlev..." files) and move, # rename, and create symlinks to them. # cd_vrfy "${postprd_dir}" basetime=$( date --date "$yyyymmdd $hh" +%y%j%H%M ) symlink_suffix="_${basetime}f${fhr}${post_mn}" -fids=( "bgdawp" "bgrd3d" ) +fids=( "prslev" "natlev" ) for fid in "${fids[@]}"; do FID="${fid^^}" post_orig_fn="${FID}.${post_fn_suffix}" diff --git a/ush/Python/plot_allvars.py b/ush/Python/plot_allvars.py index 3c12f58901..dd4e29786f 100755 --- a/ush/Python/plot_allvars.py +++ b/ush/Python/plot_allvars.py @@ -261,7 +261,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): vtime = ndate(itime,int(fhr)) # Define the location of the input file - data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') # Get the lats and lons grids = [data1] diff --git a/ush/Python/plot_allvars_diff.py b/ush/Python/plot_allvars_diff.py index ae0d718c6b..9eddb5e109 100755 --- a/ush/Python/plot_allvars_diff.py +++ b/ush/Python/plot_allvars_diff.py @@ -267,8 +267,8 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): # Define the location of the input files - data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') - data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.bgdawpf'+fhour+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') + data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') # Get the lats and lons grids = [data1, data2] diff --git a/ush/templates/parm/metplus/APCP_01h.conf b/ush/templates/parm/metplus/APCP_01h.conf index 01b1edc58e..9c23ce1eeb 100644 --- a/ush/templates/parm/metplus/APCP_01h.conf +++ b/ush/templates/parm/metplus/APCP_01h.conf @@ -14,7 +14,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} [filename_templates] # format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 diff --git a/ush/templates/parm/metplus/APCP_03h.conf b/ush/templates/parm/metplus/APCP_03h.conf index 21a7d059a1..d76687a7da 100644 --- a/ush/templates/parm/metplus/APCP_03h.conf +++ b/ush/templates/parm/metplus/APCP_03h.conf @@ -19,8 +19,8 @@ LOG_DIR = {OUTPUT_BASE}/log [filename_templates] # format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # ANLYS diff --git a/ush/templates/parm/metplus/APCP_06h.conf b/ush/templates/parm/metplus/APCP_06h.conf index 20ec8ec31b..4150e5eb8a 100644 --- a/ush/templates/parm/metplus/APCP_06h.conf +++ b/ush/templates/parm/metplus/APCP_06h.conf @@ -19,8 +19,8 @@ LOG_DIR = {OUTPUT_BASE}/log [filename_templates] # format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # ANLYS diff --git a/ush/templates/parm/metplus/APCP_24h.conf b/ush/templates/parm/metplus/APCP_24h.conf index 046eca2b37..acd3fd5a03 100644 --- a/ush/templates/parm/metplus/APCP_24h.conf +++ b/ush/templates/parm/metplus/APCP_24h.conf @@ -19,8 +19,8 @@ LOG_DIR = {OUTPUT_BASE}/log [filename_templates] # format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # ANLYS diff --git a/ush/templates/parm/metplus/GridDiag_REFC.conf b/ush/templates/parm/metplus/GridDiag_REFC.conf index d148800fb5..c71d6a1079 100755 --- a/ush/templates/parm/metplus/GridDiag_REFC.conf +++ b/ush/templates/parm/metplus/GridDiag_REFC.conf @@ -77,7 +77,7 @@ GRID_DIAG_OUTPUT_DIR = {OUTPUT_BASE} [filename_templates] -GRID_DIAG_INPUT_TEMPLATE = expt_dirs/RRFS_baseline_summer/{init?fmt=%Y%m%d%H}/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, obs_data/mrms/proc/{valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +GRID_DIAG_INPUT_TEMPLATE = expt_dirs/RRFS_baseline_summer/{init?fmt=%Y%m%d%H}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, obs_data/mrms/proc/{valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{ENV[INIT_BEG]}-{ENV[INIT_END]}_f{ENV[FHR_FIRST]}-{ENV[FHR_LAST]}.nc #when using RUN_ONCE diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 27e08d0589..1f9e3554f9 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -20,7 +20,7 @@ PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index d0be4bebf2..f9d3083caf 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -20,7 +20,7 @@ PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf index 94200119a7..6aa8d198f3 100644 --- a/ush/templates/parm/metplus/REFC.conf +++ b/ush/templates/parm/metplus/REFC.conf @@ -14,7 +14,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} [filename_templates] # format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 diff --git a/ush/templates/parm/metplus/RETOP.conf b/ush/templates/parm/metplus/RETOP.conf index 68b37d2fce..5b57d3c34c 100644 --- a/ush/templates/parm/metplus/RETOP.conf +++ b/ush/templates/parm/metplus/RETOP.conf @@ -14,7 +14,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} [filename_templates] # format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 From 715d634bbc80b2309e5753c60ce7e0958c96de3e Mon Sep 17 00:00:00 2001 From: jwolff-ncar Date: Wed, 1 Sep 2021 12:02:27 -0600 Subject: [PATCH 082/203] Port verification tasks to run on Cheyenne (#578) * Create modulefile for running vx tasks on Cheyenne * Added paths to staged data needed to run MET_verification WE2E test on Cheyenne * fix extra lines --- modulefiles/tasks/cheyenne/run_vx.local | 11 +++++++++++ tests/WE2E/run_WE2E_tests.sh | 23 ++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 modulefiles/tasks/cheyenne/run_vx.local diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local new file mode 100644 index 0000000000..234d799088 --- /dev/null +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -0,0 +1,11 @@ +#%Module + +if [module-info mode load] { + system "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib_20200427" +} + +if [module-info mode remove] { + system "deactivate" +} +module use /glade/p/ral/jntp/MET/MET_releases/modulefiles +module load met/10.0.0 diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 143be80849..14f8178966 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -950,13 +950,11 @@ external model files should be located has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi - - EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" fi - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then @@ -972,11 +970,9 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" fi - # # Make sure that the forecast length is evenly divisible by the interval # between the times at which the lateral boundary conditions will be @@ -1027,7 +1023,9 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" #----------------------------------------------------------------------- # if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then + [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then if [ "$MACHINE" = "WCOSS_CRAY" ]; then met_install_dir="/gpfs/hps/nco/ops/nwprod/met.v9.1.3" @@ -1046,9 +1044,16 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" elif [ "$MACHINE" = "HERA" ]; then met_install_dir="/contrib/met/10.0.0" metplus_path="/contrib/METplus/METplus-4.0.0" - ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ccpa/proc" - mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/mrms/proc" - ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ndas/proc" + ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc" + mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc" + ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc" + met_bin_exec="bin" + elif [ "$MACHINE" = "CHEYENNE" ]; then + met_install_dir="/glade/p/ral/jntp/MET/MET_releases/10.0.0" + metplus_path="/glade/p/ral/jntp/MET/METplus/METplus-4.0.0" + ccpa_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc" + mrms_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc" + ndas_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc" met_bin_exec="bin" else print_err_msg_exit "\ From dd9e3ab211bf48e9cb02c659877a9570e745005f Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 7 Sep 2021 11:20:03 -0400 Subject: [PATCH 083/203] Change EMC_post to UPP (#590) --- scripts/exregional_run_fcst.sh | 6 +++--- scripts/exregional_run_post.sh | 6 +++--- ush/config_defaults.sh | 2 +- ush/setup.sh | 18 +++++++++--------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index e8a7402bce..cba4499230 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -450,7 +450,7 @@ create_symlink_to_file target="${FIELD_DICT_FP}" \ relative="${relative_link_flag}" if [ ${WRITE_DOPOST} = "TRUE" ]; then - cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat + cp_vrfy ${UPP_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then post_config_fp="${CUSTOM_POST_CONFIG_FP}" print_info_msg " @@ -458,7 +458,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\" ====================================================================" else - post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt" + post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam.txt" print_info_msg " ==================================================================== post_config_fp = \"${post_config_fp}\" @@ -466,7 +466,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then fi cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt cp_vrfy ${post_config_fp} ./postxconfig-NT.txt - cp_vrfy ${EMC_POST_DIR}/parm/params_grib2_tbl_new . + cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . fi if [ "${DO_ENSEMBLE}" = TRUE ]; then diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index b722784330..d2c2ccae38 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -164,7 +164,7 @@ esac #----------------------------------------------------------------------- # rm_vrfy -f fort.* -cp_vrfy ${EMC_POST_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat +cp_vrfy ${UPP_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then post_config_fp="${CUSTOM_POST_CONFIG_FP}" print_info_msg " @@ -175,7 +175,7 @@ to the temporary work directory (tmp_dir): tmp_dir = \"${tmp_dir}\" ====================================================================" else - post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt" + post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam.txt" print_info_msg " ==================================================================== Copying the default post flat file specified by post_config_fp to the @@ -185,7 +185,7 @@ temporary work directory (tmp_dir): ====================================================================" fi cp_vrfy ${post_config_fp} ./postxconfig-NT.txt -cp_vrfy ${EMC_POST_DIR}/parm/params_grib2_tbl_new . +cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 823cf12e89..8fb02f5c91 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1514,7 +1514,7 @@ DT_SUBHOURLY_POST_MNTS="00" # should be used for post-processing the model data. If this is set to # "TRUE", then the workflow will use the custom post-processing (UPP) # configuration file specified in CUSTOM_POST_CONFIG_FP. Otherwise, a -# default configuration file provided in the EMC_post repository will be +# default configuration file provided in the UPP repository will be # used. # # CUSTOM_POST_CONFIG_FP: diff --git a/ush/setup.sh b/ush/setup.sh index 1794ab7074..ce2dacc61b 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -977,21 +977,21 @@ Please clone the external repository containing the code in this direct- ory, build the executables, and then rerun the workflow." fi # -# Get the base directory of the EMC_post code. +# Get the base directory of the UPP code. # -external_name="EMC_post" -EMC_POST_DIR=$( \ +external_name="UPP" +UPP_DIR=$( \ get_manage_externals_config_property \ "${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ print_err_msg_exit "\ Call to function get_manage_externals_config_property failed." -EMC_POST_DIR="${SR_WX_APP_TOP_DIR}/${EMC_POST_DIR}" -if [ ! -d "${EMC_POST_DIR}" ]; then +UPP_DIR="${SR_WX_APP_TOP_DIR}/${UPP_DIR}" +if [ ! -d "${UPP_DIR}" ]; then print_err_msg_exit "\ -The base directory in which the EMC_post source code should be located -(EMC_POST_DIR) does not exist: - EMS_POST_DIR = \"${EMC_POST_DIR}\" +The base directory in which the UPP source code should be located +(UPP_DIR) does not exist: + UPP_DIR = \"${UPP_DIR}\" Please clone the external repository containing the code in this directory, build the executable, and then rerun the workflow." fi @@ -2682,7 +2682,7 @@ UFS_WTHR_MDL_DIR="${UFS_WTHR_MDL_DIR}" UFS_UTILS_DIR="${UFS_UTILS_DIR}" SFC_CLIMO_INPUT_DIR="${SFC_CLIMO_INPUT_DIR}" TOPO_DIR="${TOPO_DIR}" -EMC_POST_DIR="${EMC_POST_DIR}" +UPP_DIR="${UPP_DIR}" EXPTDIR="$EXPTDIR" LOGDIR="$LOGDIR" From 226f0bb58d31fe9d6b412f8650533cc7aa93b141 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 8 Sep 2021 19:52:08 -0400 Subject: [PATCH 084/203] Remove unbound variables (#592) --- tests/WE2E/run_WE2E_tests.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 14f8178966..1e2e6db818 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -1023,9 +1023,7 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" #----------------------------------------------------------------------- # if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then + [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then if [ "$MACHINE" = "WCOSS_CRAY" ]; then met_install_dir="/gpfs/hps/nco/ops/nwprod/met.v9.1.3" From b2a853ce320be06d05065add7bf5d70141363198 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Sat, 11 Sep 2021 07:32:39 -0400 Subject: [PATCH 085/203] Update met/metplus version on wcoss cray (#595) --- modulefiles/tasks/wcoss_cray/run_vx.local | 4 ++-- tests/WE2E/run_WE2E_tests.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modulefiles/tasks/wcoss_cray/run_vx.local b/modulefiles/tasks/wcoss_cray/run_vx.local index c615e42be5..6dbf48c0f9 100644 --- a/modulefiles/tasks/wcoss_cray/run_vx.local +++ b/modulefiles/tasks/wcoss_cray/run_vx.local @@ -9,5 +9,5 @@ module load udreg module load ugni module load xpmem -module use /gpfs/hps/nco/ops/nwprod/modulefiles -module load met/9.1.3 +module use /gpfs/hps3/emc/meso/noscrub/emc.metplus/modulefiles +module load met/10.0.0 diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 1e2e6db818..3ca9a2c013 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -1026,8 +1026,8 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then if [ "$MACHINE" = "WCOSS_CRAY" ]; then - met_install_dir="/gpfs/hps/nco/ops/nwprod/met.v9.1.3" - metplus_path="/gpfs/hps/nco/ops/nwprod/metplus.v3.1.1/METplus-3.1.1" + met_install_dir="/gpfs/hps3/emc/meso/noscrub/emc.metplus/met/10.0.0" + metplus_path="/gpfs/hps3/emc/meso/noscrub/emc.metplus/METplus/METplus-4.0.0" ccpa_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" mrms_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" ndas_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" From c34ca165db75111d019017efc97e6777d7540eaa Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Wed, 15 Sep 2021 20:54:28 -0600 Subject: [PATCH 086/203] Add METplus-based ensemble verification (#575) * Fix to post flat file. * Create MET and METplus config files under ush/templates/parm * Added script to pull and reorg ccpa data. Added a script to run gridstat with METplus. Updated MET and METplus config files. * Added new jjob for running grid-stat vx. Updated setup.sh to include grid-stat vx. Updated run_gridstatvx script. * Fixed typo on script name from ksh to sh * Moved some hard coded items out from the script to the XML * Updates to get METplus to run with fewer hard-coded paths. * Updates to add grid-stat task to XML generation. * Bug fixes for adding grid-stat to XML generation * Updates to remove hard-coded paths in config files * Change log dir to put master_metplus log file with other logs under log/, rather than default logs/. * Updates to generate xml without hard-coded paths for MET * Add hera gridstat module file * Add METplus point-stat task for both sfc and upper air * Small tweaks to remove hard coded paths and add some flexibility * Updates for adding point-stat into auto-generated xml * Add in function to set point-stat task to FALSE * Final tweaks to get it to generate the xml correctly * Minor updates to run ensure 0,6,12,18 * Tweaks to var list for Point-Stat * Add METplus settings to config_defaults * Move quote for end of settings and fix extra comment. * Fix typos to populate templates correctly * Updated to include SCRIPTSDIR and other MET specific settings along with updates to FHR syntax * Update module loads on hera * Fixed comment for BOTH_VARn_THRESH to avoid syntax issues * Added files to run grid_stat for a variety of accumulation intervals, including 3, 6, and 24h * Added module load hpss * Remove module load informatino from these scripts * Updated the method of turning on/off vx tasks using jinja template if statement * Remove commented out lines of code. Fixed typo. Removed gen_wflow.out file. * Updated pull scripts to have file names dependent on date to pull from HPSS. Updated to export a few more local variables that METplus conf needed in scripts. Updated workflow to use service queue (for now) to for 1h grid_stat and point_stat run and default for 3+h accumulation grid_stat runs) * moved common_hera.conf to common.conf - no platform specific information included that needs to be handled. * Remove common_hera.conf * Add scripts to pull and process MRMS data from NOAA HPSS * Updates for REFC vx tasks * updates to obs pull scripts * Update for adding in reflectivity verification using MRMS analyses and updating name of model output to RRFS rather than HRRR * Updates to account for CCPA issues on HPSS - day off for 00-05 UTC directories * Verification mods to feature/add metplus (#1) * Remove unused/outdated code (#313) ## DESCRIPTION OF CHANGES: * In setup.sh and generate_FV3LAM_wflow.sh, remove temporary codes that fix bugs in the FV3_GFS_2017_gfdlmp_regional suite definition file because those bugs have been fixed (in the ufs-weather-model repo). * In setup.sh, remove block of code that is no longer necessary because chgres_cube can now initialize from external model data with either 4 or 9 soil levels, and run with LSMs of either 4 or 9 soil levels. * Remove modifications to LD_LIBRARY_PATH in exregional_run_fcst.sh. * For the make_ics and make_lbcs tasks, move the setting of APRUN and other machine-specific actions from the J-job to the ex-script in order to be consistent with the other workflow tasks. * Fix indentation and edit comments. * Remove unused file load_fv3gfs_modules.sh. ## TESTS CONDUCTED: Ran two WE2E tests on hera, new_ESGgrid and new_GFDLgrid: * new_ESGgrid uses the FV3_GFS_2017_gfdlmp_regional suite. The test was successful. * new_GFDLgrid uses the FV3_GFS_2017_gfdlmp suite. The test was successful. ## ISSUE (optional): This resolves issue #198. * Add and call a function that checks for use of Thompson microphysics parameterization in the SDF and if so, adjusts certain workflow arrays to contain the names and other associated values of the fixed files needed by this parameterization so that those files are automatically copied and/or linked to. (#319) ## DESCRIPTION OF CHANGES: Add and call a function that checks for use of Thompson microphysics parameterization in the suite definition file (SDF). If not, do nothing. If so, add to the appropriate workflow arrays the names and other associated values of the fixed files needed by this parameterization so that they are automatically copied and/or linked to instead of being regenerated from scratch in the run_fcst task. ## TESTS CONDUCTED: On hera, ran two WE2E tests, one in NCO mode (nco_RRFS_CONUS_25km_HRRRX_RAPX) and the other in community mode (suite_FV3_GSD_v0). These use suites FV3_GSD_SAR and FV3_GSD_v0, respectively, and both of these call Thompson microphysics. Both succeeded. ## ISSUE (optional): This PR resolves issue #297. * RRFS_v1beta SDF changes after reverting from GSL to GFS GWD suite (#322) (#327) ## DESCRIPTION OF CHANGES: Removed checks on the RRFS_v1beta SDF implemented for use with the GSL GWD suite (now uses the GFS GWD suite). No longer copies staged orography files necessary for the GSL GWD suite. ## TESTS CONDUCTED: Runs to completion on Hera. End-to-end runs DOT_OR_USCORE and suite_FV3_RRFS_v1beta succeeded on Cheyenne. Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> * Update FV3.input.nml for fhzero = 1.0 * Updated conf files for file name conventions. * Updated MET scripts and MRMS pull scripts. * Adjust RRFS_CONUS_... grids (#294) ## DESCRIPTION OF CHANGES: * Adjust RRFS_CONUS_25km, RRFS_CONUS_13km, and RRFS_CONUS_3km grid parameters so that: * All grids, including their 4-cell-wide halos, lie completely within the HRRRX domain. * All grids have dimensions nx and ny that factor "nicely", i.e. they don't have factors greather than 7. * The write-component grids corresponding to these three native grids cover as much of the native grids as possible without going outside of the native grid boundaries. The updated NCL scripts (see below) were used to generate the write-component grid parameters. * For the RRFS_CONUS_13km grid, reduce the time step (DT_ATMOS) from 180sec to 45sec. This is necessary to get a successful forecast with the GSD_SAR suite, and thus likely also the RRFS_v1beta suite. * Modify WE2E testing system as follows: * Add new tests with the RRFS_CONUS_25km, RRFS_CONUS_13km, and RRFS_CONUS_3km grids that use the GFS_v15p2 and RRFS_v1beta suites (which are now the ones officially supported in the first release of the short-range weather app) instead of the GFS_v16beta and GSD_SAR suites, respectively. * For clarity, rename the test configuration files that use the GFS_v16beta and GSD_SAR suites so they include the suite name. * Update list of WE2E tests (baselines_list.txt). * Update the NCL plotting scripts to be able to plot grids with the latest version of the workflow. ## TESTS CONDUCTED: On hera, ran tests with all three grids with the GFS_v15p2 and RRFS_v1beta suites (a total of 6 tests). All were successful. * Remove redundant model_configure.${CCPP_PHYS_SUITE} template files; use Jinja2 to create model_configure (#321) ## DESCRIPTION OF CHANGES: * Remove model_configure template files whose names depend on the physics suite, i.e. files with names of the form model_configure.${CCPP_PHYS_SUITE}. Only a single template file is needed because the contents of the model_configure file are not suite dependent. This leaves just one template file (named model_configure). * Change the function create_model_configure_file.sh and the template file model_configure so they use jinja2 instead of sed to replace placeholder values. * Absorb the contents of the write-component template files wrtcmp_lambert_conformal, wrtcmp_regional_latlon, and wrtcmp_rotated_latlon into the new jinja2-compliant model_configure file. We can do this because Jinja2 allows use of if-statements in the template file. * In the new model_configure jinja2 template file, include comments to explain the various write-component parameters. ## TESTS CONDUCTED: On Hera, ran the two WE2E tests new_ESGgrid and new_GFDLgrid. The first uses a "lambert_conformal" type of write-component grid, and the second uses a "rotated_latlon" type of write-component grid. (The write-component also allows "regional_latlon" type grids, which is just the usual earth-relative latlon coordinate system, but we do not have any cases that use that.) Both tests succeeded. ## ISSUE (optional): This PR resolves issue #281. * Add Thompson ice- and water-friendly aerosol climo file support (#332) * Add if statement in set_thompson_mp_fix_files.sh to source Thompson climo file when using a combination of a Thompson-based SDF and non-RAP/HRRR external model data * Modify if statement based on external models for Thompson climo file * Remove workflow variable EMC_GRID_NAME (#333) ## DESCRIPTION OF CHANGES: * Remove the workflow variable EMC_GRID_NAME. Henceforth, PREDEF_GRID_NAME is the only variable that can be used to set the name of the predefined grid to use. * Make appropriate change of variable name (EMC_GRID_NAME --> PREDEF_GRID_NAME) in the WE2E test configuration files. * Change anywhere the "conus" and "conus_c96" grids are specified to "EMC_CONUS_3km" and "EMC_CONUS_coarse", respectively. * Rename WE2E test configuration files with names containing the strings "conus" and "conus_c96" by replacing these strings with "EMC_CONUS_3km" and "EMC_CONUS_coarse", respectively. * Update the list of WE2E test names (tests/baselines_list.txt). * Bug fixes not directly related to grids: * In config.nco.sh, remove settings of QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST since these are now set automatically (due to another PR). * In the template file FV3LAM_wflow.xml, add the ensemble member name after RUN_FCST_TN in the dependency of the run_post metatask. ## TESTS CONDUCTED: Since this change only affects runs in NCO mode, the following NCO-mode WE2E tests were rerun on hera, all successfully: ``` nco_EMC_CONUS_3km SUCCESS nco_EMC_CONUS_coarse SUCCESS nco_EMC_CONUS_coarse__suite_FV3_GFS_2017_gfdlmp SUCCESS nco_RRFS_CONUS_25km_HRRRX_RAPX SUCCESS nco_RRFS_CONUS_3km_FV3GFS_FV3GFS SUCCESS nco_RRFS_CONUS_3km_HRRRX_RAPX SUCCESS nco_ensemble SUCCESS ``` * Port workflow to Orion (#309) ## DESCRIPTION OF CHANGES: * Add stanzas for Orion where necessary. * Add new module files for Orion. * On Orion, both the slurm partition and the slurm QOS need to be specified in the rocoto XML in order to be able to have wall times longer than 30 mins (the partition needs to be specified because it is by default "debug", which has a limit of 30 mins). Thus, introduce modifications to more easily specify slurm partitions: * Remove the workflow variables QUEUE_DEFAULT_TAG, QUEUE_HPSS_TAG, and QUEUE_FCST_TAG that are currently used to determine whether QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST specify the names of queue/QOS's or slurm partitions. * Add the workflow variables PARTITION_DEFAULT_TAG, PARTITION_HPSS_TAG, and PARTITION_FCST_TAG. These will be used to specify slurm partitions only, and the variables QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST will be used to specify queues/QOS's only. IMPORTANT NOTE: On Orion, in order to load the regional_workflow environment needed for generating an experiment, the user must first issue the following commands: ``` module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles module load miniconda3 conda activate regional_workflow ``` ## TESTS CONDUCTED: Ran 11 WE2E tests on Orion, Hera, and Cheyenne. Results on Orion: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km FAILURE - In the run_fcst task. * Error message: !!! (1) Error in subr radiation_aerosols: unrealistic surface pressure = 1 NaN new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` Results on Hera: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km SUCCESS new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` Results on Cheyenne: ``` community_ensemble_2mems SUCCESS DOT_OR_USCORE SUCCESS grid_GSD_HRRR_AK_50km FAILURE - In run_fcst task. * Error message: !!! (1) Error in subr radiation_aerosols: unrealistic surface pressure = 1 NaN new_ESGgrid SUCCESS new_GFDLgrid SUCCESS regional_001 SUCCESS regional_002 SUCCESS suite_FV3_GFS_v15p2 SUCCESS suite_FV3_GFS_v16beta SUCCESS suite_FV3_GSD_SAR SUCCESS suite_FV3_GSD_v0 SUCCESS ``` All succeed except GSD_HRRR_AK_50km on Orion and Cheyenne. It is not clear why grid_GSD_HRRR_AK_50km fails on Orion and Cheyenne but not Hera. Seems to point to a bug in the forecast model. These two failures are not so important since this grid will soon be deprecated. Also tested successfully on Jet by @JeffBeck-NOAA and on Odin and Stampede by @ywangwof. ## ISSUE: This resolves Issue #152. ## CONTRIBUTORS: @JeffBeck-NOAA @ywangwof @christinaholtNOAA * Removed comments from exregional_get_mrms_files.sh and removed fhzero from FV3.input.yml * Update FV3.input.nml for fhzero = 1.0 * Updated conf files for file name conventions. * Updated MET scripts and MRMS pull scripts. * Removed comments from exregional_get_mrms_files.sh and removed fhzero from FV3.input.yml Co-authored-by: gsketefian <31046882+gsketefian@users.noreply.github.com> Co-authored-by: Michael Kavulich Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Co-authored-by: Jamie Wolff * Change cov_thresh for REFL to be a true max in nbrhood as SPC does. * Job script for get_obs_ccpa * Jobs script for get_obs_mrms * Jobs script for get_obs_ndas * Added external variables necessary to get_ccpa script * Updated workflow template with separate get obs tasks * Separated pull scripts from run scripts * Added necessary defaults/values for defining pull tasks * Added module files, default config.sh options, and changed dependencies for vx tasks * Changed name of new workflow to FV3LAM_wflow.xml * Added task get_obs_tn, removed config.sh, updated config_defaults and config.community.sh * Adjusted the community and default config files based on comments * Updated FV3LAM workflow * Fixed discrepancies in config.community.sh * Fixed discrepancies in config_defaults.sh * Fixed discrepancies in config_defaults.sh round 2 * Fixed discrepancies in config_defaults.sh round 3 * Fixed discrepancies in config_defaults.sh round 4 * Fixed discrepancies in config.community.sh round 2 * Fixed discrepancies in config.community.sh round 3 * Fixed discrepancies in generate_FV3LAM_wflow.sh * Fixed discrepancies in generate_FV3LAM_wflow.sh round 2 * Fixed discrepancies in generate_FV3LAM_wflow.sh round 3 * Updated FV3LAM_wflow template * Separated Pull Data Scripts from Run Vx Scripts: Feature/add_metplus (#2) * Job script for get_obs_ccpa * Jobs script for get_obs_mrms * Jobs script for get_obs_ndas * Added external variables necessary to get_ccpa script * Updated workflow template with separate get obs tasks * Separated pull scripts from run scripts * Added necessary defaults/values for defining pull tasks * Added module files, default config.sh options, and changed dependencies for vx tasks * Changed name of new workflow to FV3LAM_wflow.xml * Added task get_obs_tn, removed config.sh, updated config_defaults and config.community.sh * Adjusted the community and default config files based on comments * Updated FV3LAM workflow * Fixed discrepancies in config.community.sh * Fixed discrepancies in config_defaults.sh * Fixed discrepancies in config_defaults.sh round 2 * Fixed discrepancies in config_defaults.sh round 3 * Fixed discrepancies in config_defaults.sh round 4 * Fixed discrepancies in config.community.sh round 2 * Fixed discrepancies in config.community.sh round 3 * Fixed discrepancies in generate_FV3LAM_wflow.sh * Fixed discrepancies in generate_FV3LAM_wflow.sh round 2 * Fixed discrepancies in generate_FV3LAM_wflow.sh round 3 * Updated FV3LAM_wflow template * Fixed the dependencies of the vx tasks * Fixed Vx Task Dependencies in Workflow: Feature/add metplus (#3) * Job script for get_obs_ccpa * Jobs script for get_obs_mrms * Jobs script for get_obs_ndas * Added external variables necessary to get_ccpa script * Updated workflow template with separate get obs tasks * Separated pull scripts from run scripts * Added necessary defaults/values for defining pull tasks * Added module files, default config.sh options, and changed dependencies for vx tasks * Changed name of new workflow to FV3LAM_wflow.xml * Added task get_obs_tn, removed config.sh, updated config_defaults and config.community.sh * Adjusted the community and default config files based on comments * Updated FV3LAM workflow * Fixed discrepancies in config.community.sh * Fixed discrepancies in config_defaults.sh * Fixed discrepancies in config_defaults.sh round 2 * Fixed discrepancies in config_defaults.sh round 3 * Fixed discrepancies in config_defaults.sh round 4 * Fixed discrepancies in config.community.sh round 2 * Fixed discrepancies in config.community.sh round 3 * Fixed discrepancies in generate_FV3LAM_wflow.sh * Fixed discrepancies in generate_FV3LAM_wflow.sh round 2 * Fixed discrepancies in generate_FV3LAM_wflow.sh round 3 * Updated FV3LAM_wflow template * Fixed the dependencies of the vx tasks * Manual merge with develop that didn't seem to work before. Trying to get feature branch updated so it will run again! * Add local module files * Add environment variable for SCRIPTSDIR * Remove echo statement * Remove old module files * Update to config_default for walltime for ndas pull. Update to metplus parm for obs file template. Update to FV3LAM xml to not include 00 hour for verification * Update template to remove full path * Verification channges for obs. (#4) * Verification channges for obs. * Update config_defaults.sh for vx description * Update config_defaults.sh to remove extraneous MET info. Co-authored-by: Michelle Harrold * Initial METplus .confs and MET config files for EnsembleStat APCP * J-Job script for running ensemble stat * Exregional script for ensemble-stat * Added EnsembleStat.conf for A6 and A24. Added PCPCombine to A3, A6, and A24. * Added EnsembleStatConfig files for 6 and 24h * Copy of workflow template with precipitation ensemble tasks added. Will become main template when testing is complete * Added export statement for number of ensemble members * Added necessary task definitions in ush * Updated workflow to included ENTITY definitions for ensstat * Fixed typo * Added ens vx configs * Pull in updates from develop that were not merging properly. Small change to config.community to turn off vx tasks by default. * Added/mod files for point ens vx. * Updated metplus conf files for ens point vx * Did manual merge of these files because it was not handled properly automatically * Adding additional variables to METplus for regional workflow (#5) * Changes made based on meeting with Michelle and Jamie * Updating fork * Cleanup after merge * Added additional ens vx * Ensemble point vx mods * Additional updates for ens and det vx * ensgrid_mean and ensgrid_prob .conf files for APCP * Updates for ensemble vx. * Added mean and prob point-stat configs * Updates to ensgrid_vx * Updates for mean/prob vx. * Updates to FV3LAM_wflow.xml * Deterministic and ensembel vx updates. * Ensgrid mean * Update setup.sh * Changed workflow template title * Updates to deterministic and ensemble verification * Created EnsembleStat METplus conf and MET config files for REFC * Added reflectivity mean and prob METplus and MET config files. Updated APCP mean and prob METplus and MET config files. * Added all J-job scripts, exregional scripts, and necessary definitons for workflow generation for all ensgrid_mean and ensgrid_prob tasks * Updates to workflow to add ensgrid_vx * Changes I made to account for runtime errors. * Made changes to directory structures * Made changes to directory structures and variables * Changed log files and stage dir. * Changes for grid- and point-vx. * Updated METplus ensemble precip conf files. * Mods for ensemble and deterministic vx. * Change to GridStatConfig_REFC_mean * Updated EnsembleStat_REFC.conf * Updated to METv10.0.0 * Updated conf files for paths. * Updated FV3LAM_wflow.xml template. * Mods for vx dependencies * Updated for censor thresh in METplus conf files; changes to FV3LAM_wflow.xml after sync with develop. * Updated exregional_run_fcst.sh generate_FV3LAM_wflow.sh to address merge with develop. * Mods for ensemble precip vx, handling padded/non-padded ensemble member names, fixes for python environment for obs pull. * Changes to RETOP (units) and REFC (naming and level) verification. * Fix OUTPUT_BASE for deterministic vx. * Changes to some verification ex-scripts for syntax and path fixes. Included start end dates of incorrect 01-h CCPA data. Removed some extra lines in FV3LAM_wflow.xml template. * Changed comp. ref. variable name in GridStat_REFC_prob.conf * Changed comp. ref. level in GridStat_REFC_prob.conf * Updated logic for number padding in the directory name when running in ensemble mode. * Added MET ensemble vx WE2E test. * Modified location of obs to live outside cycle dir, allowing for obs to be shared across cycles. * Mods to address comments on PR575. * Updated ensemble METPlus conf files for changes to post output name. * Addessed comments in PR and mods for 10-m WIND. * Addressing final comments in PR. Co-authored-by: Jamie Wolff Co-authored-by: gsketefian <31046882+gsketefian@users.noreply.github.com> Co-authored-by: Michael Kavulich Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Co-authored-by: lindsayrblank Co-authored-by: Michelle Harrold Co-authored-by: PerryShafran-NOAA <62255233+PerryShafran-NOAA@users.noreply.github.com> --- jobs/JREGIONAL_RUN_VX_ENSGRID | 90 ++ jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 90 ++ jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 90 ++ jobs/JREGIONAL_RUN_VX_ENSPOINT | 88 ++ jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 88 ++ jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 88 ++ jobs/JREGIONAL_RUN_VX_GRIDSTAT | 40 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 40 +- scripts/exregional_get_ccpa_files.sh | 17 +- scripts/exregional_get_mrms_files.sh | 2 +- scripts/exregional_get_ndas_files.sh | 2 +- scripts/exregional_run_ensgridvx.sh | 180 ++++ scripts/exregional_run_ensgridvx_mean.sh | 189 +++++ scripts/exregional_run_ensgridvx_prob.sh | 189 +++++ scripts/exregional_run_enspointvx.sh | 170 ++++ scripts/exregional_run_enspointvx_mean.sh | 170 ++++ scripts/exregional_run_enspointvx_prob.sh | 170 ++++ scripts/exregional_run_gridstatvx.sh | 54 +- scripts/exregional_run_pointstatvx.sh | 36 +- tests/WE2E/run_WE2E_tests.sh | 5 +- .../config.MET_ensemble_verification.sh | 38 + ush/config.community.sh | 5 + ush/config_defaults.sh | 70 +- ush/generate_FV3LAM_wflow.sh | 58 ++ ush/setup.sh | 64 +- ush/templates/FV3LAM_wflow.xml | 778 ++++++++++++++++-- .../parm/met/EnsembleStatConfig_APCP | 257 ++++++ .../parm/met/EnsembleStatConfig_REFC | 265 ++++++ .../parm/met/EnsembleStatConfig_point | 290 +++++++ .../parm/met/GridStatConfig_APCP_mean | 178 ++++ .../parm/met/GridStatConfig_APCP_prob | 178 ++++ ...ridStatConfig_REFL => GridStatConfig_REFC} | 0 .../parm/met/GridStatConfig_REFC_mean | 219 +++++ .../parm/met/GridStatConfig_REFC_prob | 219 +++++ ush/templates/parm/met/PointStatConfig | 5 - ush/templates/parm/met/PointStatConfig_mean | 254 ++++++ ush/templates/parm/met/PointStatConfig_prob | 254 ++++++ ush/templates/parm/metplus/APCP_01h.conf | 18 +- ush/templates/parm/metplus/APCP_03h.conf | 23 +- ush/templates/parm/metplus/APCP_06h.conf | 22 +- ush/templates/parm/metplus/APCP_24h.conf | 22 +- .../parm/metplus/EnsembleStat_APCP01h.conf | 129 +++ .../parm/metplus/EnsembleStat_APCP03h.conf | 172 ++++ .../parm/metplus/EnsembleStat_APCP06h.conf | 172 ++++ .../parm/metplus/EnsembleStat_APCP24h.conf | 172 ++++ .../parm/metplus/EnsembleStat_REFC.conf | 119 +++ .../parm/metplus/EnsembleStat_RETOP.conf | 118 +++ .../parm/metplus/EnsembleStat_conus_sfc.conf | 170 ++++ .../parm/metplus/EnsembleStat_upper_air.conf | 173 ++++ .../parm/metplus/GridStat_APCP01h_mean.conf | 95 +++ .../parm/metplus/GridStat_APCP01h_prob.conf | 129 +++ .../parm/metplus/GridStat_APCP03h_mean.conf | 89 ++ .../parm/metplus/GridStat_APCP03h_prob.conf | 138 ++++ .../parm/metplus/GridStat_APCP06h_mean.conf | 89 ++ .../parm/metplus/GridStat_APCP06h_prob.conf | 138 ++++ .../parm/metplus/GridStat_APCP24h_mean.conf | 89 ++ .../parm/metplus/GridStat_APCP24h_prob.conf | 138 ++++ .../parm/metplus/GridStat_REFC_mean.conf | 94 +++ .../parm/metplus/GridStat_REFC_prob.conf | 122 +++ .../parm/metplus/GridStat_RETOP_mean.conf | 95 +++ .../parm/metplus/GridStat_RETOP_prob.conf | 122 +++ .../parm/metplus/PointStat_conus_sfc.conf | 67 +- .../metplus/PointStat_conus_sfc_mean.conf | 151 ++++ .../metplus/PointStat_conus_sfc_prob.conf | 195 +++++ .../parm/metplus/PointStat_upper_air.conf | 80 +- .../metplus/PointStat_upper_air_mean.conf | 152 ++++ .../metplus/PointStat_upper_air_prob.conf | 196 +++++ ush/templates/parm/metplus/REFC.conf | 20 +- ush/templates/parm/metplus/RETOP.conf | 20 +- ush/valid_param_vals.sh | 4 +- 70 files changed, 8162 insertions(+), 301 deletions(-) create mode 100755 jobs/JREGIONAL_RUN_VX_ENSGRID create mode 100755 jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN create mode 100755 jobs/JREGIONAL_RUN_VX_ENSGRID_PROB create mode 100755 jobs/JREGIONAL_RUN_VX_ENSPOINT create mode 100755 jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN create mode 100755 jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB create mode 100755 scripts/exregional_run_ensgridvx.sh create mode 100755 scripts/exregional_run_ensgridvx_mean.sh create mode 100755 scripts/exregional_run_ensgridvx_prob.sh create mode 100755 scripts/exregional_run_enspointvx.sh create mode 100755 scripts/exregional_run_enspointvx_mean.sh create mode 100755 scripts/exregional_run_enspointvx_prob.sh create mode 100644 tests/WE2E/test_configs/wflow_features/config.MET_ensemble_verification.sh create mode 100755 ush/templates/parm/met/EnsembleStatConfig_APCP create mode 100755 ush/templates/parm/met/EnsembleStatConfig_REFC create mode 100644 ush/templates/parm/met/EnsembleStatConfig_point create mode 100755 ush/templates/parm/met/GridStatConfig_APCP_mean create mode 100755 ush/templates/parm/met/GridStatConfig_APCP_prob rename ush/templates/parm/met/{GridStatConfig_REFL => GridStatConfig_REFC} (100%) create mode 100644 ush/templates/parm/met/GridStatConfig_REFC_mean create mode 100644 ush/templates/parm/met/GridStatConfig_REFC_prob create mode 100644 ush/templates/parm/met/PointStatConfig_mean create mode 100644 ush/templates/parm/met/PointStatConfig_prob create mode 100644 ush/templates/parm/metplus/EnsembleStat_APCP01h.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_APCP03h.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_APCP06h.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_APCP24h.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_REFC.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_RETOP.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf create mode 100644 ush/templates/parm/metplus/EnsembleStat_upper_air.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP01h_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP01h_prob.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP03h_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP03h_prob.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP06h_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP06h_prob.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP24h_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP24h_prob.conf create mode 100644 ush/templates/parm/metplus/GridStat_REFC_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_REFC_prob.conf create mode 100644 ush/templates/parm/metplus/GridStat_RETOP_mean.conf create mode 100644 ush/templates/parm/metplus/GridStat_RETOP_prob.conf create mode 100644 ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf create mode 100644 ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf create mode 100644 ush/templates/parm/metplus/PointStat_upper_air_mean.conf create mode 100644 ush/templates/parm/metplus/PointStat_upper_air_prob.conf diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID new file mode 100755 index 0000000000..9fa88c2f7d --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -0,0 +1,90 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script runs METplus for ensemble-stat on the UPP output files by +# initialization time for all forecast hours for gridded analysis. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for ensemble-stat +on gridded fields by initialization time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_ensgridvx.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN new file mode 100755 index 0000000000..67945ca7b9 --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -0,0 +1,90 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script runs METplus for grid-stat on the UPP output files by +# initialization time for all forecast hours for gridded analysis. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for grid-stat +on gridded fields by initialization time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_ensgridvx_mean.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB new file mode 100755 index 0000000000..f792d4b77f --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -0,0 +1,90 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script runs METplus for grid-stat on the UPP output files by +# initialization time for all forecast hours for gridded analysis. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for grid-stat +on gridded fields by initialization time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_ensgridvx_prob.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT new file mode 100755 index 0000000000..dac8873a2a --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -0,0 +1,88 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for point-stat +by initialitation time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_enspointvx.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN new file mode 100755 index 0000000000..3ae1cb56dc --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -0,0 +1,88 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for point-stat +by initialitation time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_enspointvx_mean.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB new file mode 100755 index 0000000000..8810d0bcbc --- /dev/null +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -0,0 +1,88 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs METplus for point-stat +by initialitation time for all forecast hours. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_enspointvx_prob.sh \ + cycle_dir="${CYCLE_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index ca29e2d931..300c79f9c6 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -57,51 +57,13 @@ by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -# If it doesn't already exist, create the directory (vx_dir) in -# which to store grid-stat output. -# -# Note that there may be a preexisting version of vx_dir from previous -# runs of this script (e.g. from the workflow task that runs this script -# failing and then being called again). Thus, we first make sure -# preexisting versions are deleted. -# -#----------------------------------------------------------------------- -# - -if [ "${RUN_ENVIR}" = "nco" ]; then - COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" - postprd_dir="$COMOUT" -else - postprd_dir="${CYCLE_DIR}/postprd" -fi -mkdir_vrfy -p "${postprd_dir}" - -if [ "${RUN_ENVIR}" = "nco" ]; then - COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" - vx_dir="$COMOUT" -else - vx_dir="${CYCLE_DIR}/metprd" -fi -mkdir_vrfy -p "${vx_dir}" - -gridstat_dir="${vx_dir}/grid_stat" -#check_for_preexist_dir "${gridstat_dir}" "delete" -mkdir_vrfy -p "${gridstat_dir}" - -cd_vrfy "${gridstat_dir}" -# -#----------------------------------------------------------------------- -# # Call the ex-script for this J-job and pass to it the necessary varia- # bles. # #----------------------------------------------------------------------- # $SCRIPTSDIR/exregional_run_gridstatvx.sh \ - cycle_dir="${CYCLE_DIR}" \ - postprd_dir="${postprd_dir}" \ - vx_dir="${vx_dir}" \ - gridstat_dir="${gridstat_dir}" || \ + cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index f5c35253a2..f67cb55479 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -55,51 +55,13 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -# If it doesn't already exist, create the directory (vx_dir) in -# which to store point-stat output. -# -# Note that there may be a preexisting version of vx_dir from previous -# runs of this script (e.g. from the workflow task that runs this script -# failing and then being called again). Thus, we first make sure -# preexisting versions are deleted. -# -#----------------------------------------------------------------------- -# - -if [ "${RUN_ENVIR}" = "nco" ]; then - COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" - postprd_dir="$COMOUT" -else - postprd_dir="${CYCLE_DIR}/postprd" -fi -mkdir_vrfy -p "${postprd_dir}" - -if [ "${RUN_ENVIR}" = "nco" ]; then - COMOUT="$COMROOT/$NET/$envir/$RUN.$PDY/$cyc" - vx_dir="$COMOUT" -else - vx_dir="${CYCLE_DIR}/metprd" -fi -mkdir_vrfy -p "${vx_dir}" - -pointstat_dir="${vx_dir}/point_stat" -#check_for_preexist_dir "${pointstat_dir}" "delete" -mkdir_vrfy -p "${pointstat_dir}" - -cd_vrfy "${pointstat_dir}" -# -#----------------------------------------------------------------------- -# # Call the ex-script for this J-job and pass to it the necessary varia- # bles. # #----------------------------------------------------------------------- # $SCRIPTSDIR/exregional_run_pointstatvx.sh \ - cycle_dir="${CYCLE_DIR}" \ - postprd_dir="${postprd_dir}" \ - vx_dir="${vx_dir}" \ - pointstat_dir="${pointstat_dir}" || \ + cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/scripts/exregional_get_ccpa_files.sh b/scripts/exregional_get_ccpa_files.sh index 2354938f1a..3c1504292d 100755 --- a/scripts/exregional_get_ccpa_files.sh +++ b/scripts/exregional_get_ccpa_files.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This script reorganizes the CCPA data into a more intuitive structure: # A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. @@ -150,7 +150,8 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` fi - # One hour CCPA files have incorrect metadeta in the files under the "00" directory. After data is pulled, reorganize into correct valid yyyymmdd structure. + # One hour CCPA files have incorrect metadeta in the files under the "00" directory from 20180718 to 20210504. + # After data is pulled, reorganize into correct valid yyyymmdd structure. if [[ ${vhh_noZero} -ge 1 && ${vhh_noZero} -le 6 ]]; then cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -ge 7 && ${vhh_noZero} -le 12 ]]; then @@ -158,9 +159,17 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do elif [[ ${vhh_noZero} -ge 13 && ${vhh_noZero} -le 18 ]]; then cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then - wgrib2 $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + else + cp $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi elif [[ ${vhh_noZero} -eq 0 ]]; then - wgrib2 $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + else + cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi fi elif [[ ${accum} == "03" ]]; then diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index ff09c55462..8819959ecc 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash . ${GLOBAL_VAR_DEFNS_FP} diff --git a/scripts/exregional_get_ndas_files.sh b/scripts/exregional_get_ndas_files.sh index f79acf77f2..e1ec7f8d68 100755 --- a/scripts/exregional_get_ndas_files.sh +++ b/scripts/exregional_get_ndas_files.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # This script reorganizes the NDAS data into a more intuitive structure: # A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh new file mode 100755 index 0000000000..ce06547530 --- /dev/null +++ b/scripts/exregional_run_ensgridvx.sh @@ -0,0 +1,180 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for ensemble-stat on +the UPP output files by initialization time for all forecast hours for +gridded data. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args + +#----------------------------------------------------------------------- +# +# Begin grid-to-grid ensemble vx. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting ensemble-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Read in FHR and create a comma-separated list +# for METplus to run over. Determine the number padding needed based +# on number of ensemble members. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +NUM_PAD=${NDIGITS_ENSMEM_NAMES} + +# +#----------------------------------------------------------------------- +# +# Create LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# + +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_${CDATE}_${VAR} +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML +# +#----------------------------------------------------------------------- +# +export SCRIPTSDIR +export EXPTDIR +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET +export NUM_ENS_MEMBERS +export NUM_PAD +export LOG_SUFFIX + +# +#----------------------------------------------------------------------- +# +# Run METplus +# +#----------------------------------------------------------------------- +# +if [ ${VAR} == "APCP" ]; then + acc="${ACCUM}h" # for stats output prefix in EnsembleStatConfig + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/EnsembleStat_${VAR}${acc}.conf +else + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/EnsembleStat_${VAR}.conf +fi + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus ensemble-stat grid completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh new file mode 100755 index 0000000000..1a4d68aa85 --- /dev/null +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -0,0 +1,189 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for grid-stat on +the UPP output files by initialization time for all forecast hours for +gridded data. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args + +#----------------------------------------------------------------------- +# +# Begin grid-to-grid vx on ensemble output. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting grid-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR} +fi + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML +# +#----------------------------------------------------------------------- +# +export SCRIPTSDIR +export EXPTDIR +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET +export INPUT_BASE +export LOG_SUFFIX + +# +#----------------------------------------------------------------------- +# +# Run METplus +# +#----------------------------------------------------------------------- +# +if [ ${VAR} == "APCP" ]; then + export acc="${ACCUM}h" + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/GridStat_${VAR}${acc}_mean.conf +else + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/GridStat_${VAR}_mean.conf +fi + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus grid-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh new file mode 100755 index 0000000000..d0c948e333 --- /dev/null +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -0,0 +1,189 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for grid-stat on +the UPP output files by initialization time for all forecast hours for +gridded data. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args + +#----------------------------------------------------------------------- +# +# Begin grid-to-grid vx on ensemble output. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting grid-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR} +fi + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML +# +#----------------------------------------------------------------------- +# +export SCRIPTSDIR +export INPUT_BASE +export EXPTDIR +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET +export LOG_SUFFIX + +# +#----------------------------------------------------------------------- +# +# Run METplus +# +#----------------------------------------------------------------------- +# +if [ ${VAR} == "APCP" ]; then + export acc="${ACCUM}h" + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/GridStat_${VAR}${acc}_prob.conf +else + ${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/GridStat_${VAR}_prob.conf +fi + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus grid-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh new file mode 100755 index 0000000000..67f0017a3b --- /dev/null +++ b/scripts/exregional_run_enspointvx.sh @@ -0,0 +1,170 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for point-stat on +the UPP output files by initialization time for all forecast hours. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args +#----------------------------------------------------------------------- +# +# Begin grid-to-point ensemble vx. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting point-based ensemble-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Create LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +LOG_SUFFIX=enspoint_${CDATE} + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML and run METplus +# +#----------------------------------------------------------------------- +# +export EXPTDIR +export LOG_SUFFIX +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET +export NUM_ENS_MEMBERS + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/EnsembleStat_conus_sfc.conf + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/EnsembleStat_upper_air.conf + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus ensemble-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh new file mode 100755 index 0000000000..ad673428a8 --- /dev/null +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -0,0 +1,170 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for point-stat on +the UPP output files by initialization time for all forecast hours. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args +#----------------------------------------------------------------------- +# +# Begin grid-to-point vx on ensemble output. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting point-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat +LOG_SUFFIX=enspoint_mean_${CDATE} + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML and run METplus +# +#----------------------------------------------------------------------- +# +export EXPTDIR +export INPUT_BASE +export LOG_SUFFIX +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_conus_sfc_mean.conf + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_upper_air_mean.conf +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus ensemble-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh new file mode 100755 index 0000000000..afecf2ff16 --- /dev/null +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -0,0 +1,170 @@ +#!/bin/bash +set -x + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs METplus for point-stat on +the UPP output files by initialization time for all forecast hours. +========================================================================" + +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args +#----------------------------------------------------------------------- +# +# Begin grid-to-point vx on ensemble output. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" "Starting point-stat verification" + +# +#----------------------------------------------------------------------- +# +# Get the cycle date and hour (in formats of yyyymmdd and hh, respect- +# ively) from CDATE. Also read in FHR and create a comma-separated list +# for METplus to run over. +# +#----------------------------------------------------------------------- +# +yyyymmdd=${CDATE:0:8} +hh=${CDATE:8:2} +cyc=$hh +export CDATE +export hh + +fhr_last=`echo ${FHR} | awk '{ print $NF }'` +export fhr_last + +fhr_list=`echo ${FHR} | sed "s/ /,/g"` +export fhr_list + +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat +LOG_SUFFIX=enspoint_prob_${CDATE} + +# +#----------------------------------------------------------------------- +# +# Check for existence of top-level OBS_DIR +# +#----------------------------------------------------------------------- +# +if [[ ! -d "$OBS_DIR" ]]; then + print_err_msg_exit "\ + Exiting: OBS_DIR does not exist." +fi + +# +#----------------------------------------------------------------------- +# +# Export some environment variables passed in by the XML and run METplus +# +#----------------------------------------------------------------------- +# +export EXPTDIR +export INPUT_BASE +export LOG_SUFFIX +export MET_INSTALL_DIR +export MET_BIN_EXEC +export METPLUS_PATH +export METPLUS_CONF +export MET_CONFIG +export MODEL +export NET + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_conus_sfc_prob.conf + +${METPLUS_PATH}/ush/master_metplus.py \ + -c ${METPLUS_CONF}/common.conf \ + -c ${METPLUS_CONF}/PointStat_upper_air_prob.conf +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +METplus ensemble-stat completed successfully. + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh index 6b377f8c43..16b641f33f 100755 --- a/scripts/exregional_run_gridstatvx.sh +++ b/scripts/exregional_run_gridstatvx.sh @@ -1,4 +1,4 @@ -#!/bin/sh -l +#!/bin/bash set -x # @@ -57,7 +57,7 @@ the UPP output files by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -valid_args=( "cycle_dir" "postprd_dir" "vx_dir" "gridstat_dir" ) +valid_args=( "cycle_dir" ) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -70,16 +70,6 @@ process_args valid_args "$@" # print_input_args valid_args -#----------------------------------------------------------------------- -# -# Remove any files from previous runs and stage necessary files in gridstat_dir. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" "Starting grid-stat verification" - -cd ${gridstat_dir} - # #----------------------------------------------------------------------- # @@ -101,6 +91,33 @@ export fhr_last fhr_list=`echo ${FHR} | sed "s/ /,/g"` export fhr_list +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then + INPUT_BASE=${EXPTDIR}/${CDATE}/postprd + OUTPUT_BASE=${EXPTDIR}/${CDATE} + if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=gridstat_${CDATE}_${VAR}_${ACCUM}h + else + LOG_SUFFIX=gridstat_${CDATE}_${VAR} + fi +elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd + OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} + ENSMEM=`echo ${SLASH_ENSMEM_SUBDIR} | cut -d"/" -f2` + MODEL=${MODEL}_${ENSMEM} + if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=gridstat_${CDATE}_${ENSMEM}_${VAR}_${ACCUM}h + else + LOG_SUFFIX=gridstat_${CDATE}_${ENSMEM}_${VAR} + fi +fi + # #----------------------------------------------------------------------- # @@ -123,13 +140,14 @@ fi # export SCRIPTSDIR export EXPTDIR +export INPUT_BASE +export OUTPUT_BASE +export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC export METPLUS_PATH export METPLUS_CONF export MET_CONFIG -export OBS_DIR -export VAR export MODEL export NET @@ -145,16 +163,10 @@ if [ ${VAR} == "APCP" ]; then ${METPLUS_PATH}/ush/master_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/${VAR}_${acc}.conf -elif [ ${VAR} == "REFC" ]; then - ${METPLUS_PATH}/ush/master_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${METPLUS_CONF}/${VAR}.conf -elif [ ${VAR} == "RETOP" ]; then +else ${METPLUS_PATH}/ush/master_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/${VAR}.conf -else - echo "No variable defined" fi # diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh index 73572d9a6f..fd68dfca8f 100755 --- a/scripts/exregional_run_pointstatvx.sh +++ b/scripts/exregional_run_pointstatvx.sh @@ -1,4 +1,4 @@ -#!/bin/sh -l +#!/bin/bash set -x # @@ -57,7 +57,7 @@ the UPP output files by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -valid_args=( "cycle_dir" "postprd_dir" "vx_dir" "pointstat_dir" ) +valid_args=( "cycle_dir" ) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -69,15 +69,6 @@ process_args valid_args "$@" #----------------------------------------------------------------------- # print_input_args valid_args -#----------------------------------------------------------------------- -# -# Remove any files from previous runs and stage necessary files in pointstat_dir. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" "Starting point-stat verification" - -cd ${pointstat_dir} # #----------------------------------------------------------------------- @@ -100,6 +91,25 @@ export fhr_last fhr_list=`echo ${FHR} | sed "s/ /,/g"` export fhr_list +# +#----------------------------------------------------------------------- +# +# Create INPUT_BASE to read into METplus conf files. +# +#----------------------------------------------------------------------- +# +if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then + INPUT_BASE=${EXPTDIR}/${CDATE}/postprd + OUTPUT_BASE=${EXPTDIR}/${CDATE} + LOG_SUFFIX=pointstat_${CDATE} +elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd + OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} + ENSMEM=`echo ${SLASH_ENSMEM_SUBDIR} | cut -d"/" -f2` + MODEL=${MODEL}_${ENSMEM} + LOG_SUFFIX=pointstat_${CDATE}_${ENSMEM} +fi + # #----------------------------------------------------------------------- # @@ -120,12 +130,14 @@ fi #----------------------------------------------------------------------- # export EXPTDIR +export INPUT_BASE +export OUTPUT_BASE +export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC export METPLUS_PATH export METPLUS_CONF export MET_CONFIG -export OBS_DIR export MODEL export NET diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 3ca9a2c013..b8eaeec74f 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -950,7 +950,6 @@ external model files should be located has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi - EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" @@ -1023,7 +1022,9 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" #----------------------------------------------------------------------- # if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then + [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then if [ "$MACHINE" = "WCOSS_CRAY" ]; then met_install_dir="/gpfs/hps3/emc/meso/noscrub/emc.metplus/met/10.0.0" diff --git a/tests/WE2E/test_configs/wflow_features/config.MET_ensemble_verification.sh b/tests/WE2E/test_configs/wflow_features/config.MET_ensemble_verification.sh new file mode 100644 index 0000000000..078ce144f9 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.MET_ensemble_verification.sh @@ -0,0 +1,38 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully with MET verification. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="2" + +USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +MODEL="FV3_GFS_v15p2_CONUS_25km" +RUN_TASK_VX_GRIDSTAT="TRUE" +RUN_TASK_VX_POINTSTAT="TRUE" +RUN_TASK_VX_ENSGRID="TRUE" +RUN_TASK_VX_ENSPOINT="TRUE" + +WTIME_RUN_FCST="01:00:00" diff --git a/ush/config.community.sh b/ush/config.community.sh index b0c1f21f1b..70433d5973 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -11,6 +11,9 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" +DO_ENSEMBLE="FALSE" +NUM_ENS_MEMBERS="2" + CCPP_PHYS_SUITE="FV3_GFS_v15p2" FCST_LEN_HRS="48" LBC_SPEC_INTVL_HRS="6" @@ -42,6 +45,8 @@ RUN_TASK_GET_OBS_MRMS="FALSE" RUN_TASK_GET_OBS_NDAS="FALSE" RUN_TASK_VX_GRIDSTAT="FALSE" RUN_TASK_VX_POINTSTAT="FALSE" +RUN_TASK_VX_ENSGRID="FALSE" +RUN_TASK_VX_ENSPOINT="FALSE" # # Uncomment the following line in order to use user-staged external model diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 8fb02f5c91..72c758df4a 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -648,7 +648,6 @@ NOMADS_file_type="nemsio" # directory or the cycle directories under it. # #----------------------------------------------------------------------- -# CCPP_PHYS_SUITE="FV3_GFS_v15p2" # #----------------------------------------------------------------------- @@ -1148,6 +1147,25 @@ VX_GRIDSTAT_03h_TN="run_gridstatvx_03h" VX_GRIDSTAT_06h_TN="run_gridstatvx_06h" VX_GRIDSTAT_24h_TN="run_gridstatvx_24h" VX_POINTSTAT_TN="run_pointstatvx" +VX_ENSGRID_TN="run_ensgridvx" +VX_ENSGRID_03h_TN="run_ensgridvx_03h" +VX_ENSGRID_06h_TN="run_ensgridvx_06h" +VX_ENSGRID_24h_TN="run_ensgridvx_24h" +VX_ENSGRID_REFC_TN="run_ensgridvx_refc" +VX_ENSGRID_RETOP_TN="run_ensgridvx_retop" +VX_ENSGRID_MEAN_TN="run_ensgridvx_mean" +VX_ENSGRID_PROB_TN="run_ensgridvx_prob" +VX_ENSGRID_MEAN_03h_TN="run_ensgridvx_mean_03h" +VX_ENSGRID_PROB_03h_TN="run_ensgridvx_prob_03h" +VX_ENSGRID_MEAN_06h_TN="run_ensgridvx_mean_06h" +VX_ENSGRID_PROB_06h_TN="run_ensgridvx_prob_06h" +VX_ENSGRID_MEAN_24h_TN="run_ensgridvx_mean_24h" +VX_ENSGRID_PROB_24h_TN="run_ensgridvx_prob_24h" +VX_ENSGRID_PROB_REFC_TN="run_ensgridvx_prob_refc" +VX_ENSGRID_PROB_RETOP_TN="run_ensgridvx_prob_retop" +VX_ENSPOINT_TN="run_enspointvx" +VX_ENSPOINT_MEAN_TN="run_enspointvx_mean" +VX_ENSPOINT_PROB_TN="run_enspointvx_prob" # #----------------------------------------------------------------------- # @@ -1191,6 +1209,15 @@ VX_POINTSTAT_TN="run_pointstatvx" # Flag that determines whether the point-stat verification task is to be # run. # +# RUN_TASK_VX_ENSGRID: +# Flag that determines whether the ensemble-stat verification for gridded +# data task is to be run. +# +# RUN_TASK_VX_ENSPOINT: +# Flag that determines whether the ensemble point verification task is +# to be run. If this flag is set, both ensemble-stat point verification +# and point verification of ensemble-stat output is computed. +# #----------------------------------------------------------------------- # RUN_TASK_MAKE_GRID="TRUE" @@ -1213,6 +1240,10 @@ RUN_TASK_GET_OBS_NDAS="FALSE" RUN_TASK_VX_GRIDSTAT="FALSE" RUN_TASK_VX_POINTSTAT="FALSE" + +RUN_TASK_VX_ENSGRID="FALSE" + +RUN_TASK_VX_ENSPOINT="FALSE" # #----------------------------------------------------------------------- # @@ -1421,6 +1452,12 @@ NNODES_GET_OBS_MRMS="1" NNODES_GET_OBS_NDAS="1" NNODES_VX_GRIDSTAT="1" NNODES_VX_POINTSTAT="1" +NNODES_VX_ENSGRID="1" +NNODES_VX_ENSGRID_MEAN="1" +NNODES_VX_ENSGRID_PROB="1" +NNODES_VX_ENSPOINT="1" +NNODES_VX_ENSPOINT_MEAN="1" +NNODES_VX_ENSPOINT_PROB="1" # # Number of MPI processes per node. # @@ -1438,6 +1475,12 @@ PPN_GET_OBS_MRMS="1" PPN_GET_OBS_NDAS="1" PPN_VX_GRIDSTAT="1" PPN_VX_POINTSTAT="1" +PPN_VX_ENSGRID="1" +PPN_VX_ENSGRID_MEAN="1" +PPN_VX_ENSGRID_PROB="1" +PPN_VX_ENSPOINT="1" +PPN_VX_ENSPOINT_MEAN="1" +PPN_VX_ENSPOINT_PROB="1" # # Walltimes. # @@ -1455,6 +1498,12 @@ WTIME_GET_OBS_MRMS="00:45:00" WTIME_GET_OBS_NDAS="02:00:00" WTIME_VX_GRIDSTAT="02:00:00" WTIME_VX_POINTSTAT="01:00:00" +WTIME_VX_ENSGRID="01:00:00" +WTIME_VX_ENSGRID_MEAN="01:00:00" +WTIME_VX_ENSGRID_PROB="01:00:00" +WTIME_VX_ENSPOINT="01:00:00" +WTIME_VX_ENSPOINT_MEAN="01:00:00" +WTIME_VX_ENSPOINT_PROB="01:00:00" # # Maximum number of attempts. # @@ -1477,6 +1526,25 @@ MAXTRIES_VX_GRIDSTAT_03h="1" MAXTRIES_VX_GRIDSTAT_06h="1" MAXTRIES_VX_GRIDSTAT_24h="1" MAXTRIES_VX_POINTSTAT="1" +MAXTRIES_VX_ENSGRID="1" +MAXTRIES_VX_ENSGRID_REFC="1" +MAXTRIES_VX_ENSGRID_RETOP="1" +MAXTRIES_VX_ENSGRID_03h="1" +MAXTRIES_VX_ENSGRID_06h="1" +MAXTRIES_VX_ENSGRID_24h="1" +MAXTRIES_VX_ENSGRID_MEAN="1" +MAXTRIES_VX_ENSGRID_PROB="1" +MAXTRIES_VX_ENSGRID_MEAN_03h="1" +MAXTRIES_VX_ENSGRID_PROB_03h="1" +MAXTRIES_VX_ENSGRID_MEAN_06h="1" +MAXTRIES_VX_ENSGRID_PROB_06h="1" +MAXTRIES_VX_ENSGRID_MEAN_24h="1" +MAXTRIES_VX_ENSGRID_PROB_24h="1" +MAXTRIES_VX_ENSGRID_PROB_REFC="1" +MAXTRIES_VX_ENSGRID_PROB_RETOP="1" +MAXTRIES_VX_ENSPOINT="1" +MAXTRIES_VX_ENSPOINT_MEAN="1" +MAXTRIES_VX_ENSPOINT_PROB="1" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 07aee40365..e5d2f496ed 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -193,6 +193,25 @@ settings="\ 'vx_gridstat_06h_tn': ${VX_GRIDSTAT_06h_TN} 'vx_gridstat_24h_tn': ${VX_GRIDSTAT_24h_TN} 'vx_pointstat_tn': ${VX_POINTSTAT_TN} + 'vx_ensgrid_tn': ${VX_ENSGRID_TN} + 'vx_ensgrid_refc_tn': ${VX_ENSGRID_REFC_TN} + 'vx_ensgrid_retop_tn': ${VX_ENSGRID_RETOP_TN} + 'vx_ensgrid_03h_tn': ${VX_ENSGRID_03h_TN} + 'vx_ensgrid_06h_tn': ${VX_ENSGRID_06h_TN} + 'vx_ensgrid_24h_tn': ${VX_ENSGRID_24h_TN} + 'vx_ensgrid_mean_tn': ${VX_ENSGRID_MEAN_TN} + 'vx_ensgrid_prob_tn': ${VX_ENSGRID_PROB_TN} + 'vx_ensgrid_mean_03h_tn': ${VX_ENSGRID_MEAN_03h_TN} + 'vx_ensgrid_prob_03h_tn': ${VX_ENSGRID_PROB_03h_TN} + 'vx_ensgrid_mean_06h_tn': ${VX_ENSGRID_MEAN_06h_TN} + 'vx_ensgrid_prob_06h_tn': ${VX_ENSGRID_PROB_06h_TN} + 'vx_ensgrid_mean_24h_tn': ${VX_ENSGRID_MEAN_24h_TN} + 'vx_ensgrid_prob_24h_tn': ${VX_ENSGRID_PROB_24h_TN} + 'vx_ensgrid_prob_refc_tn': ${VX_ENSGRID_PROB_REFC_TN} + 'vx_ensgrid_prob_retop_tn': ${VX_ENSGRID_PROB_RETOP_TN} + 'vx_enspoint_tn': ${VX_ENSPOINT_TN} + 'vx_enspoint_mean_tn': ${VX_ENSPOINT_MEAN_TN} + 'vx_enspoint_prob_tn': ${VX_ENSPOINT_PROB_TN} # # Entity used to load the module file for each GET_OBS_* task. # @@ -214,6 +233,12 @@ settings="\ 'nnodes_get_obs_ndas': ${NNODES_GET_OBS_NDAS} 'nnodes_vx_gridstat': ${NNODES_VX_GRIDSTAT} 'nnodes_vx_pointstat': ${NNODES_VX_POINTSTAT} + 'nnodes_vx_ensgrid': ${NNODES_VX_ENSGRID} + 'nnodes_vx_ensgrid_mean': ${NNODES_VX_ENSGRID_MEAN} + 'nnodes_vx_ensgrid_prob': ${NNODES_VX_ENSGRID_PROB} + 'nnodes_vx_enspoint': ${NNODES_VX_ENSPOINT} + 'nnodes_vx_enspoint_mean': ${NNODES_VX_ENSPOINT_MEAN} + 'nnodes_vx_enspoint_prob': ${NNODES_VX_ENSPOINT_PROB} # # Number of cores used for a task # @@ -237,6 +262,12 @@ settings="\ 'ppn_get_obs_ndas': ${PPN_GET_OBS_NDAS} 'ppn_vx_gridstat': ${PPN_VX_GRIDSTAT} 'ppn_vx_pointstat': ${PPN_VX_POINTSTAT} + 'ppn_vx_ensgrid': ${PPN_VX_ENSGRID} + 'ppn_vx_ensgrid_mean': ${PPN_VX_ENSGRID_MEAN} + 'ppn_vx_ensgrid_prob': ${PPN_VX_ENSGRID_PROB} + 'ppn_vx_enspoint': ${PPN_VX_ENSPOINT} + 'ppn_vx_enspoint_mean': ${PPN_VX_ENSPOINT_MEAN} + 'ppn_vx_enspoint_prob': ${PPN_VX_ENSPOINT_PROB} # # Maximum wallclock time for each task. # @@ -254,6 +285,12 @@ settings="\ 'wtime_get_obs_ndas': ${WTIME_GET_OBS_NDAS} 'wtime_vx_gridstat': ${WTIME_VX_GRIDSTAT} 'wtime_vx_pointstat': ${WTIME_VX_POINTSTAT} + 'wtime_vx_ensgrid': ${WTIME_VX_ENSGRID} + 'wtime_vx_ensgrid_mean': ${WTIME_VX_ENSGRID_MEAN} + 'wtime_vx_ensgrid_prob': ${WTIME_VX_ENSGRID_PROB} + 'wtime_vx_enspoint': ${WTIME_VX_ENSPOINT} + 'wtime_vx_enspoint_mean': ${WTIME_VX_ENSPOINT_MEAN} + 'wtime_vx_enspoint_prob': ${WTIME_VX_ENSPOINT_PROB} # # Maximum number of tries for each task. # @@ -276,6 +313,25 @@ settings="\ 'maxtries_vx_gridstat_06h': ${MAXTRIES_VX_GRIDSTAT_06h} 'maxtries_vx_gridstat_24h': ${MAXTRIES_VX_GRIDSTAT_24h} 'maxtries_vx_pointstat': ${MAXTRIES_VX_POINTSTAT} + 'maxtries_vx_ensgrid': ${MAXTRIES_VX_ENSGRID} + 'maxtries_vx_ensgrid_refc': ${MAXTRIES_VX_ENSGRID_REFC} + 'maxtries_vx_ensgrid_retop': ${MAXTRIES_VX_ENSGRID_RETOP} + 'maxtries_vx_ensgrid_03h': ${MAXTRIES_VX_ENSGRID_03h} + 'maxtries_vx_ensgrid_06h': ${MAXTRIES_VX_ENSGRID_06h} + 'maxtries_vx_ensgrid_24h': ${MAXTRIES_VX_ENSGRID_24h} + 'maxtries_vx_ensgrid_mean': ${MAXTRIES_VX_ENSGRID_MEAN} + 'maxtries_vx_ensgrid_prob': ${MAXTRIES_VX_ENSGRID_PROB} + 'maxtries_vx_ensgrid_mean_03h': ${MAXTRIES_VX_ENSGRID_MEAN_03h} + 'maxtries_vx_ensgrid_prob_03h': ${MAXTRIES_VX_ENSGRID_PROB_03h} + 'maxtries_vx_ensgrid_mean_06h': ${MAXTRIES_VX_ENSGRID_MEAN_06h} + 'maxtries_vx_ensgrid_prob_06h': ${MAXTRIES_VX_ENSGRID_PROB_06h} + 'maxtries_vx_ensgrid_mean_24h': ${MAXTRIES_VX_ENSGRID_MEAN_24h} + 'maxtries_vx_ensgrid_prob_24h': ${MAXTRIES_VX_ENSGRID_PROB_24h} + 'maxtries_vx_ensgrid_prob_refc': ${MAXTRIES_VX_ENSGRID_PROB_REFC} + 'maxtries_vx_ensgrid_prob_retop': ${MAXTRIES_VX_ENSGRID_PROB_RETOP} + 'maxtries_vx_enspoint': ${MAXTRIES_VX_ENSPOINT} + 'maxtries_vx_enspoint_mean': ${MAXTRIES_VX_ENSPOINT_MEAN} + 'maxtries_vx_enspoint_prob': ${MAXTRIES_VX_ENSPOINT_PROB} # # Flags that specify whether to run the preprocessing or # verification-related tasks. @@ -289,6 +345,8 @@ settings="\ 'run_task_get_obs_ndas': ${RUN_TASK_GET_OBS_NDAS} 'run_task_vx_gridstat': ${RUN_TASK_VX_GRIDSTAT} 'run_task_vx_pointstat': ${RUN_TASK_VX_POINTSTAT} + 'run_task_vx_ensgrid': ${RUN_TASK_VX_ENSGRID} + 'run_task_vx_enspoint': ${RUN_TASK_VX_ENSPOINT} # # Number of physical cores per node for the current machine. # diff --git a/ush/setup.sh b/ush/setup.sh index ce2dacc61b..bc11c67b40 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -290,6 +290,49 @@ elif [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ] || \ [ "${RUN_TASK_VX_POINTSTAT}" = "NO" ]; then RUN_TASK_VX_POINTSTAT="FALSE" fi + +# +#----------------------------------------------------------------------- +# +# Make sure that RUN_TASK_VX_ENSGRID is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_RUN_TASK_VX_ENSGRID" +# +# Set RUN_TASK_VX_ENSGRID to either "TRUE" or "FALSE" so we don't have to +# consider other valid values later on. +# +RUN_TASK_VX_ENSGRID=${RUN_TASK_VX_ENSGRID^^} +if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSGRID}" = "YES" ]; then + RUN_TASK_VX_ENSGRID="TRUE" +elif [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ] || \ + [ "${RUN_TASK_VX_ENSGRID}" = "NO" ]; then + RUN_TASK_VX_ENSGRID="FALSE" +fi + +# +# +#----------------------------------------------------------------------- +# +# Make sure that RUN_TASK_VX_ENSPOINT is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_RUN_TASK_VX_ENSPOINT" +# +# Set RUN_TASK_VX_ENSPOINT to either "TRUE" or "FALSE" so we don't have to +# consider other valid values later on. +# +RUN_TASK_VX_ENSPOINT=${RUN_TASK_VX_ENSPOINT^^} +if [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSPOINT}" = "YES" ]; then + RUN_TASK_VX_ENSPOINT="TRUE" +elif [ "${RUN_TASK_VX_ENSPOINT}" = "FALSE" ] || \ + [ "${RUN_TASK_VX_ENSPOINT}" = "NO" ]; then + RUN_TASK_VX_ENSPOINT="FALSE" +fi # #----------------------------------------------------------------------- # @@ -1723,7 +1766,6 @@ Reset values are: msg=" When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated orography files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} where @@ -1834,6 +1876,26 @@ Reset value is:" fi + if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ + [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ]; then + + msg=" +When RUN_ENVIR is set to \"nco\", it is assumed that the verification +will not be run. + RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" +Resetting RUN_TASK_VX_ENSGRID to \"FALSE\" +Reset value is:" + + RUN_TASK_VX_ENSGRID="FALSE" + + msg="$msg"" + RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" +" + + print_info_msg "$msg" + + fi + # #----------------------------------------------------------------------- # diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 5198552665..f7e7b94715 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,26 +22,45 @@ Parameters needed by the job scheduler. - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + @@ -666,7 +685,6 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - &RSRV_HPSS; @@ -692,14 +710,6 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} ACCUM01 - - {%- if write_dopost %} - - {%- else %} - - {%- endif %} - - {%- endif %} @@ -708,7 +718,6 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - &RSRV_HPSS; @@ -735,14 +744,6 @@ the tag to be identical to the ones above for other output times. SCRIPTSDIR&SCRIPTSDIR; VARREFC RETOP - - {%- if write_dopost %} - - {%- else %} - - {%- endif %} - - {%- endif %} @@ -751,7 +752,6 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - &RSRV_HPSS; @@ -776,14 +776,6 @@ the tag to be identical to the ones above for other output times. cyc@H FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} - - {%- if write_dopost %} - - {%- else %} - - {%- endif %} - - {%- endif %} @@ -792,8 +784,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -801,7 +792,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_TN; - &LOGDIR;/&VX_GRIDSTAT_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -812,10 +803,20 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM01 + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} {%- if run_task_get_obs_ccpa %} - + + + {%- if write_dopost %} + + {%- else %} + + {%- endif %} + {%- else %} {%- if write_dopost %} @@ -833,7 +834,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -841,7 +842,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_REFC_TN; - &LOGDIR;/&VX_GRIDSTAT_REFC_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -851,10 +852,20 @@ the tag to be identical to the ones above for other output times. cyc@H FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARREFC + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} {%- if run_task_get_obs_mrms %} - + + + {%- if write_dopost %} + + {%- else %} + + {%- endif %} + {%- else %} {%- if write_dopost %} @@ -872,7 +883,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -880,7 +891,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_RETOP_TN; - &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -890,10 +901,20 @@ the tag to be identical to the ones above for other output times. cyc@H FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARRETOP + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} {%- if run_task_get_obs_mrms %} - + + + {%- if write_dopost %} + + {%- else %} + + {%- endif %} + {%- else %} {%- if write_dopost %} @@ -906,12 +927,12 @@ the tag to be identical to the ones above for other output times. {%- endif %} -{%- if run_task_vx_gridstat %} +{%- if run_task_vx_gridstat and fcst_len_hrs >= 3 %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -919,7 +940,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_03h_TN; - &LOGDIR;/&VX_GRIDSTAT_03h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -930,20 +951,23 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM03 + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} - + {%- endif %} -{%- if run_task_vx_gridstat %} +{%- if run_task_vx_gridstat and fcst_len_hrs >= 6 %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -951,7 +975,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_06h_TN; - &LOGDIR;/&VX_GRIDSTAT_06h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -962,20 +986,23 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM06 + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} - + {%- endif %} -{%- if run_task_vx_gridstat %} +{%- if run_task_vx_gridstat and fcst_len_hrs >= 24 %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -983,7 +1010,7 @@ the tag to be identical to the ones above for other output times. {{ wtime_vx_gridstat }} &NCORES_PER_NODE; &VX_GRIDSTAT_24h_TN; - &LOGDIR;/&VX_GRIDSTAT_24h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -994,9 +1021,12 @@ the tag to be identical to the ones above for other output times. FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM24 + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} - + @@ -1007,15 +1037,15 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - - + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_POINTSTAT" {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} {{ wtime_vx_pointstat }} &NCORES_PER_NODE; &VX_POINTSTAT_TN; - &LOGDIR;/&VX_POINTSTAT_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; @@ -1023,11 +1053,21 @@ the tag to be identical to the ones above for other output times. CDATE@Y@m@d@H PDY@Y@m@d cyc@H - FHR {%- for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + {%- if do_ensemble %} + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + {%- endif %} {%- if run_task_get_obs_ndas %} - + + + {%- if write_dopost %} + + {%- else %} + + {%- endif %} + {%- else %} {%- if write_dopost %} @@ -1038,9 +1078,593 @@ the tag to be identical to the ones above for other output times. - {%- endif %} + {%- if do_ensemble %} {%- endif %} + +{%- if run_task_vx_ensgrid %} + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_TN; + &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM01 + + + + + + + +{%- if fcst_len_hrs >= 3 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_03h_TN; + &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM03 + + + + + + +{%- endif %} +{%- if fcst_len_hrs >= 6 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_06h_TN; + &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM06 + + + + + + +{%- endif %} +{%- if fcst_len_hrs >= 24 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_24h_TN; + &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM24 + + + + + + +{%- endif %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_REFC_TN; + &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARREFC + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" + {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} + {{ wtime_vx_ensgrid }} + &NCORES_PER_NODE; + &VX_ENSGRID_RETOP_TN; + &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARRETOP + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" + {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} + {{ wtime_vx_ensgrid_mean }} + &NCORES_PER_NODE; + &VX_ENSGRID_MEAN_TN; + &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM01 + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_TN; + &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM01 + + + + + + +{%- if fcst_len_hrs >= 3 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" + {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} + {{ wtime_vx_ensgrid_mean }} + &NCORES_PER_NODE; + &VX_ENSGRID_MEAN_03h_TN; + &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM03 + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_03h_TN; + &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM03 + + + + + + +{%- endif %} +{%- if fcst_len_hrs >= 6 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" + {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} + {{ wtime_vx_ensgrid_mean }} + &NCORES_PER_NODE; + &VX_ENSGRID_MEAN_06h_TN; + &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM06 + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_06h_TN; + &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM06 + + + + + + +{%- endif %} +{%- if fcst_len_hrs >= 24 %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" + {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} + {{ wtime_vx_ensgrid_mean }} + &NCORES_PER_NODE; + &VX_ENSGRID_MEAN_24h_TN; + &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM24 + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_24h_TN; + &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&CCPA_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} + VARAPCP + ACCUM24 + + + + + + +{%- endif %} + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_REFC_TN; + &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARREFC + + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" + {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} + {{ wtime_vx_ensgrid_prob }} + &NCORES_PER_NODE; + &VX_ENSGRID_PROB_RETOP_TN; + &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&MRMS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + VARRETOP + + + + + + +{%- endif %} + +{%- if run_task_vx_enspoint %} + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT" + {{ nnodes_vx_enspoint }}:ppn={{ ppn_vx_enspoint }} + {{ wtime_vx_enspoint }} + &NCORES_PER_NODE; + &VX_ENSPOINT_TN; + &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&NDAS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT_MEAN" + {{ nnodes_vx_enspoint_mean }}:ppn={{ ppn_vx_enspoint_mean }} + {{ wtime_vx_enspoint_mean }} + &NCORES_PER_NODE; + &VX_ENSPOINT_MEAN_TN; + &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&NDAS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + + + + + + + + + + + &RSRV_DEFAULT; + &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT_PROB" + {{ nnodes_vx_enspoint_prob }}:ppn={{ ppn_vx_enspoint_prob }} + {{ wtime_vx_enspoint_prob }} + &NCORES_PER_NODE; + &VX_ENSPOINT_PROB_TN; + &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + OBS_DIR&NDAS_OBS_DIR; + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CDATE@Y@m@d@H + PDY@Y@m@d + cyc@H + FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} + + + + + + + +{%- endif %} diff --git a/ush/templates/parm/met/EnsembleStatConfig_APCP b/ush/templates/parm/met/EnsembleStatConfig_APCP new file mode 100755 index 0000000000..34d560cb6a --- /dev/null +++ b/ush/templates/parm/met/EnsembleStatConfig_APCP @@ -0,0 +1,257 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Ensemble-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BUDGET; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product fields to be processed +// +ens = { + ens_thresh = ${ENS_THRESH}; + vld_thresh = 1.0; + + field = [ ${ENS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality = []; +duplicate_flag = UNIQUE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = TRUE; + +// +// Observation error options +// Set dist_type to NONE to use the observation error table instead +// May be set separately in each "obs.field" entry +// +obs_error = { + flag = FALSE; // TRUE or FALSE + dist_type = NONE; // Distribution type + dist_parm = []; // Distribution parameters + inst_bias_scale = 1.0; // Instrument bias scale adjustment + inst_bias_offset = 0.0; // Instrument bias offset adjustment + min = NA; // Valid range of data + max = NA; +} + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + +// +// Ensemble bin sizes +// May be set separately in each "obs.field" entry +// +ens_ssvar_bin_size = 1.0; +ens_phist_bin_size = 0.05; + +// +// Categorical thresholds to define ensemble probabilities +// May be set separately in each "fcst.field" entry +// +prob_cat_thresh = []; + + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 31; + hour_interval = 6; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +// +// May be set separately in each "obs.field" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + write_bins = TRUE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + ${METPLUS_MASK_GRID} + ${METPLUS_MASK_POLY} + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.01 ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + type = [ { method = NEAREST; width = 1; } ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + ecnt = NONE; + rps = NONE; + rhist = STAT; + phist = STAT; + orank = NONE; + ssvar = STAT; + relp = STAT; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product output types +// +ensemble_flag = { + latlon = TRUE; + mean = TRUE; + stdev = TRUE; + minus = FALSE; + plus = FALSE; + min = FALSE; + max = FALSE; + range = FALSE; + vld_count = TRUE; + frequency = TRUE; + nep = FALSE; + nmep = FALSE; + rank = FALSE; + weight = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Random number generator +// +rng = { + type = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/EnsembleStatConfig_REFC b/ush/templates/parm/met/EnsembleStatConfig_REFC new file mode 100755 index 0000000000..5d44356b63 --- /dev/null +++ b/ush/templates/parm/met/EnsembleStatConfig_REFC @@ -0,0 +1,265 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Ensemble-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BUDGET; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product fields to be processed +// + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + +ens = { + ens_thresh = ${ENS_THRESH}; + vld_thresh = 1.0; + + field = [ ${ENS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation fields to be verified +// + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality = []; +duplicate_flag = UNIQUE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = TRUE; + +// +// Observation error options +// Set dist_type to NONE to use the observation error table instead +// May be set separately in each "obs.field" entry +// +obs_error = { + flag = FALSE; // TRUE or FALSE + dist_type = NONE; // Distribution type + dist_parm = []; // Distribution parameters + inst_bias_scale = 1.0; // Instrument bias scale adjustment + inst_bias_offset = 0.0; // Instrument bias offset adjustment + min = NA; // Valid range of data + max = NA; +} + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + +// +// Ensemble bin sizes +// May be set separately in each "obs.field" entry +// +ens_ssvar_bin_size = 1.0; +ens_phist_bin_size = 0.05; + +// +// Categorical thresholds to define ensemble probabilities +// May be set separately in each "fcst.field" entry +// +prob_cat_thresh = []; + + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 31; + hour_interval = 6; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +// +// May be set separately in each "obs.field" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + write_bins = TRUE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + ${METPLUS_MASK_GRID} + ${METPLUS_MASK_POLY} + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.01 ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + type = [ { method = NEAREST; width = 1; } ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + ecnt = NONE; + rps = NONE; + rhist = STAT; + phist = STAT; + orank = NONE; + ssvar = STAT; + relp = STAT; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product output types +// +ensemble_flag = { + latlon = TRUE; + mean = TRUE; + stdev = TRUE; + minus = FALSE; + plus = FALSE; + min = FALSE; + max = FALSE; + range = FALSE; + vld_count = TRUE; + frequency = TRUE; + nep = FALSE; + nmep = FALSE; + rank = FALSE; + weight = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Random number generator +// +rng = { + type = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/EnsembleStatConfig_point b/ush/templates/parm/met/EnsembleStatConfig_point new file mode 100644 index 0000000000..648893e201 --- /dev/null +++ b/ush/templates/parm/met/EnsembleStatConfig_point @@ -0,0 +1,290 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Ensemble-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product fields to be processed +// +ens = { + ens_thresh = 0.05; + vld_thresh = 1.0; + + field = [ ${ENS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood ensemble probabilities +// +nbrhd_prob = { + width = [ 5 ]; + shape = CIRCLE; + vld_thresh = 0.0; +} + +// +// NMEP smoothing methods +// +nmep_smooth = { + vld_thresh = 0.0; + shape = CIRCLE; + gaussian_dx = 81.27; + gaussian_radius = 120; + type = [ + { + method = GAUSSIAN; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; +} +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +${METPLUS_MESSAGE_TYPE} +sid_inc = []; +sid_exc = []; +obs_thresh = [ NA ]; +obs_quality = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; +skip_const = FALSE; + +// +// Observation error options +// Set dist_type to NONE to use the observation error table instead +// May be set separately in each "obs.field" entry +// +obs_error = { + flag = FALSE; // TRUE or FALSE + dist_type = NONE; // Distribution type + dist_parm = []; // Distribution parameters + inst_bias_scale = 1.0; // Instrument bias scale adjustment + inst_bias_offset = 0.0; // Instrument bias offset adjustment + min = NA; // Valid range of data + max = NA; +} + +// +// Mapping of message type group name to comma-separated list of values +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } +]; + +// +// Ensemble bin sizes +// May be set separately in each "obs.field" entry +// +ens_ssvar_bin_size = 1.0; +ens_phist_bin_size = 0.05; + +// +// Categorical thresholds to define ensemble probabilities +// May be set separately in each "fcst.field" entry +// +prob_cat_thresh = []; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = DW_MEAN; + day_interval = 31; + hour_interval = 6; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +// +// May be set separately in each "obs.field" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + write_bins = TRUE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + ${METPLUS_MASK_GRID} + ${METPLUS_MASK_POLY} + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + ecnt = STAT; + rps = NONE; + rhist = STAT; + phist = STAT; + orank = STAT; + ssvar = STAT; + relp = STAT; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Ensemble product output types +// +ensemble_flag = { + latlon = TRUE; + mean = TRUE; + stdev = TRUE; + minus = FALSE; + plus = FALSE; + min = FALSE; + max = FALSE; + range = TRUE; + vld_count = TRUE; + frequency = TRUE; + nep = FALSE; + nmep = FALSE; + rank = TRUE; + weight = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Random number generator +// +rng = { + type = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_APCP_mean b/ush/templates/parm/met/GridStatConfig_APCP_mean new file mode 100755 index 0000000000..6aa4c8cc3b --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_APCP_mean @@ -0,0 +1,178 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = BUDGET; + width = 2; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = STAT; + ctc = STAT; + cts = STAT; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + nbrctc = STAT; + nbrcts = STAT; + nbrcnt = STAT; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_APCP_prob b/ush/templates/parm/met/GridStatConfig_APCP_prob new file mode 100755 index 0000000000..d6ef8cb0ec --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_APCP_prob @@ -0,0 +1,178 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = BUDGET; + width = 2; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = BOTH; + vld_thresh = 1.0; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + pct = STAT; + pstd = STAT; + pjc = STAT; + prc = STAT; + nbrctc = NONE; + nbrcts = NONE; + nbrcnt = NONE; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFL b/ush/templates/parm/met/GridStatConfig_REFC similarity index 100% rename from ush/templates/parm/met/GridStatConfig_REFL rename to ush/templates/parm/met/GridStatConfig_REFC diff --git a/ush/templates/parm/met/GridStatConfig_REFC_mean b/ush/templates/parm/met/GridStatConfig_REFC_mean new file mode 100644 index 0000000000..ade839df7a --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_REFC_mean @@ -0,0 +1,219 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = BUDGET; + width = 2; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = NONE; + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fourier decomposition +// May be set separately in each "obs.field" entry +// +fourier = { + wave_1d_beg = []; + wave_1d_end = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Gradient statistics +// May be set separately in each "obs.field" entry +// +gradient = { + dx = [ 1 ]; + dy = [ 1 ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = STAT; + ctc = STAT; + cts = STAT; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + nbrctc = STAT; + nbrcts = STAT; + nbrcnt = STAT; + grad = NONE; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + fourier = FALSE; + gradient = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFC_prob b/ush/templates/parm/met/GridStatConfig_REFC_prob new file mode 100644 index 0000000000..61570c4938 --- /dev/null +++ b/ush/templates/parm/met/GridStatConfig_REFC_prob @@ -0,0 +1,219 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "${OBTYPE}"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + vld_thresh = 0.5; + method = BUDGET; + width = 2; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +cat_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +// +// Forecast and observation fields to be verified +// + +M_to_KFT(x) = x * 3.28084 * 0.001; +KM_to_KFT(x) = x * 3280.84 * 0.001; + +fcst = { + field = [ ${FCST_FIELD} ]; +} + +obs = { + field = [ ${OBS_FIELD} ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + vld_thresh = 0.5; + method = NEAREST; + width = 1; + } + + time_interp_method = DW_MEAN; + match_day = FALSE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = []; + poly = [ ${VERIF_MASK} ]; + sid = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = NONE; + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = NEAREST; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + field = BOTH; + shape = ${NEIGHBORHOOD_SHAPE}; + width = [ ${NEIGHBORHOOD_WIDTH} ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fourier decomposition +// May be set separately in each "obs.field" entry +// +fourier = { + wave_1d_beg = []; + wave_1d_end = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Gradient statistics +// May be set separately in each "obs.field" entry +// +gradient = { + dx = [ 1 ]; + dy = [ 1 ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = STAT; + pstd = STAT; + pjc = STAT; + prc = STAT; + eclv = NONE; + nbrctc = NONE; + nbrcts = NONE; + nbrcnt = NONE; + grad = NONE; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = { + latlon = FALSE; + raw = FALSE; + diff = FALSE; + climo = FALSE; + weight = FALSE; + nbrhd = FALSE; + fourier = FALSE; + gradient = FALSE; + apply_mask = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +rank_corr_flag = FALSE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig b/ush/templates/parm/met/PointStatConfig index 453f10c2eb..384ee83a6f 100644 --- a/ush/templates/parm/met/PointStatConfig +++ b/ush/templates/parm/met/PointStatConfig @@ -170,11 +170,6 @@ obs_window = { // mask = { grid = ${POINT_STAT_GRID}; -// poly = [ -// "${FIXverif_global}/vx_mask_files/grid2obs/CONUS.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/EAST.nc", -// "${FIXverif_global}/vx_mask_files/grid2obs/WEST.nc" -// ]; poly = ${POINT_STAT_POLY}; sid = ${POINT_STAT_STATION_ID}; llpnt = []; diff --git a/ush/templates/parm/met/PointStatConfig_mean b/ush/templates/parm/met/PointStatConfig_mean new file mode 100644 index 0000000000..0b935c5786 --- /dev/null +++ b/ush/templates/parm/met/PointStatConfig_mean @@ -0,0 +1,254 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; +rank_corr_flag = FALSE; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; + }; + +obs = { + field = [ ${OBS_FIELD} ]; + }; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = ${POINT_STAT_MESSAGE_TYPE}; +sid_exc = []; +obs_quality = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = NEAREST; + match_month = TRUE; + match_day = TRUE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; +write_cdf_bins = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Land/Sea mask +// For LANDSF message types, only use forecast grid points where land = TRUE. +// For WATERSF message types, only use forecast grid points where land = FALSE. +// land_mask.flag may be set separately in each "obs.field" entry. +// +land_mask = { + flag = FALSE; + file_name = []; + field = { name = "LAND"; level = "L0"; } + regrid = { method = NEAREST; width = 1; } + thresh = eq1; +} + +// +// Topography +// For SURFACE message types, only use observations where the topo - station +// elevation difference meets the use_obs_thresh threshold. +// For the observations kept, when interpolating forecast data to the +// observation location, only use forecast grid points where the topo - station +// difference meets the interp_fcst_thresh threshold. +// topo_mask.flag may be set separately in each "obs.field" entry. +// +topo_mask = { + flag = FALSE; + file_name = []; + field = { name = "TOPO"; level = "L0"; } + regrid = { method = BILIN; width = 2; } + use_obs_thresh = ge-100&&le100; + interp_fcst_thresh = ge-50&&le50; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.field" entry +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.field" entry +// +mask = { + grid = ${POINT_STAT_GRID}; + poly = ${POINT_STAT_POLY}; + sid = ${POINT_STAT_STATION_ID}; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.field" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// May be set separately in each "obs.field" entry +// +interp = { + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = BILIN; + width = 2; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// HiRA verification method +// May be set separately in each "obs.field" entry +// +hira = { + flag = FALSE; + width = [ 2, 3, 4, 5 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.field" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = STAT; + sl1l2 = STAT; + sal1l2 = NONE; + vl1l2 = STAT; + val1l2 = NONE; + vcnt = STAT; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_prob b/ush/templates/parm/met/PointStatConfig_prob new file mode 100644 index 0000000000..f21ba1aaad --- /dev/null +++ b/ush/templates/parm/met/PointStatConfig_prob @@ -0,0 +1,254 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Point-Stat configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "${MODEL}"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +regrid = { + to_grid = ${REGRID_TO_GRID}; + method = BILIN; + width = 2; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; +rank_corr_flag = FALSE; + +// +// Forecast and observation fields to be verified +// +fcst = { + field = [ ${FCST_FIELD} ]; + }; + +obs = { + field = [ ${OBS_FIELD} ]; + }; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation filtering options +// May be set separately in each "obs.field" entry +// +message_type = ${POINT_STAT_MESSAGE_TYPE}; +sid_exc = []; +obs_quality = []; +duplicate_flag = NONE; +obs_summary = NONE; +obs_perc_value = 50; + +// +// Mapping of message type group name to comma-separated list of values. +// +message_type_group_map = [ + { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, + { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, + { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, + { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, + { key = "LANDSF"; val = "ADPSFC,MSONET"; }, + { key = "WATERSF"; val = "SFCSHP"; } +]; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology data +// +climo_mean = { + + file_name = []; + field = []; + + regrid = { + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + time_interp_method = NEAREST; + match_month = TRUE; + match_day = TRUE; + time_step = 21600; +} + +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +climo_cdf_bins = 1; +write_cdf_bins = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Land/Sea mask +// For LANDSF message types, only use forecast grid points where land = TRUE. +// For WATERSF message types, only use forecast grid points where land = FALSE. +// land_mask.flag may be set separately in each "obs.field" entry. +// +land_mask = { + flag = FALSE; + file_name = []; + field = { name = "LAND"; level = "L0"; } + regrid = { method = NEAREST; width = 1; } + thresh = eq1; +} + +// +// Topography +// For SURFACE message types, only use observations where the topo - station +// elevation difference meets the use_obs_thresh threshold. +// For the observations kept, when interpolating forecast data to the +// observation location, only use forecast grid points where the topo - station +// difference meets the interp_fcst_thresh threshold. +// topo_mask.flag may be set separately in each "obs.field" entry. +// +topo_mask = { + flag = FALSE; + file_name = []; + field = { name = "TOPO"; level = "L0"; } + regrid = { method = BILIN; width = 2; } + use_obs_thresh = ge-100&&le100; + interp_fcst_thresh = ge-50&&le50; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.field" entry +// +obs_window = { + beg = ${OBS_WINDOW_BEGIN}; + end = ${OBS_WINDOW_END}; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.field" entry +// +mask = { + grid = ${POINT_STAT_GRID}; + poly = ${POINT_STAT_POLY}; + sid = ${POINT_STAT_STATION_ID}; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.field" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Interpolation methods +// May be set separately in each "obs.field" entry +// +interp = { + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { + method = BILIN; + width = 2; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// HiRA verification method +// May be set separately in each "obs.field" entry +// +hira = { + flag = FALSE; + width = [ 2, 3, 4, 5 ]; + vld_thresh = 1.0; + cov_thresh = [ ==0.25 ]; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.field" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = STAT; + pstd = STAT; + pjc = STAT; + prc = STAT; + eclv = NONE; + mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/metplus/APCP_01h.conf b/ush/templates/parm/metplus/APCP_01h.conf index 9c23ce1eeb..4f3599d327 100644 --- a/ush/templates/parm/metplus/APCP_01h.conf +++ b/ush/templates/parm/metplus/APCP_01h.conf @@ -1,17 +1,19 @@ -# Grid to Grid Precipitation +# Grid to Grid Precipitation Example [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h + [filename_templates] # format of filenames FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 @@ -19,11 +21,17 @@ FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_01h.conf + [config] # Model to verify MODEL = {ENV[MODEL]} diff --git a/ush/templates/parm/metplus/APCP_03h.conf b/ush/templates/parm/metplus/APCP_03h.conf index d76687a7da..ce3f1f5db9 100644 --- a/ush/templates/parm/metplus/APCP_03h.conf +++ b/ush/templates/parm/metplus/APCP_03h.conf @@ -1,21 +1,24 @@ -# Grid to Grid Precipitation +# Grid to Grid Precipitation Example [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +# Grid to Grid Precipitation FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h [filename_templates] # format of filenames @@ -28,11 +31,17 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_03h.conf + [config] # Model to verify MODEL = {ENV[MODEL]} diff --git a/ush/templates/parm/metplus/APCP_06h.conf b/ush/templates/parm/metplus/APCP_06h.conf index 4150e5eb8a..6773ddba8a 100644 --- a/ush/templates/parm/metplus/APCP_06h.conf +++ b/ush/templates/parm/metplus/APCP_06h.conf @@ -1,21 +1,23 @@ -# Grid to Grid Precipitation +# Grid to Grid Precipitation Example [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h [filename_templates] # format of filenames @@ -28,11 +30,17 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_06h.conf + [config] # Model to verify MODEL = {ENV[MODEL]} diff --git a/ush/templates/parm/metplus/APCP_24h.conf b/ush/templates/parm/metplus/APCP_24h.conf index acd3fd5a03..8b5a6bb985 100644 --- a/ush/templates/parm/metplus/APCP_24h.conf +++ b/ush/templates/parm/metplus/APCP_24h.conf @@ -1,21 +1,23 @@ -# Grid to Grid Precipitation +# Grid to Grid Precipitation Example [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h [filename_templates] # format of filenames @@ -28,11 +30,17 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_24h.conf + [config] # Model to verify MODEL = {ENV[MODEL]} diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf new file mode 100644 index 0000000000..46dd14a82e --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf @@ -0,0 +1,129 @@ +# Ensemble Stat Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h + +[filename_templates] +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_01h.conf + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} + +# Name to identify observation data in output +OBTYPE = CCPA + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = 0 +OBS_FILE_WINDOW_END = 0 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_01h_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = APCP +ENS_VAR1_LEVELS = A01 +ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A01 + +FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION +OBS_VAR1_NAME = {FCST_VAR1_NAME} +OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} + +OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf new file mode 100644 index 0000000000..cf7c95a23f --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf @@ -0,0 +1,172 @@ +# Ensemble Stat Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# Output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h + +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_03h.conf + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PcpCombine, EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# Accumulation interval available in observation data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = 0 +OBS_FILE_WINDOW_END = 0 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_03h_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = APCP +ENS_VAR1_LEVELS = A03 +ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A03 + +# FIGURE OUT WHAT BIN SIZE SHOULD BE +FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION +OBS_VAR1_NAME = {FCST_VAR1_NAME} +OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} + +OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf new file mode 100644 index 0000000000..9f8ddaafd6 --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf @@ -0,0 +1,172 @@ +# Ensemble Stat Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# Output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h + +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_06h.conf + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PcpCombine, EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# Accumulation interval available in observation data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = 0 +OBS_FILE_WINDOW_END = 0 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_06h_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = APCP +ENS_VAR1_LEVELS = A06 +ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A06 + +# FIGURE OUT WHAT BIN SIZE SHOULD BE +FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION +OBS_VAR1_NAME = {FCST_VAR1_NAME} +OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} + +OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf new file mode 100644 index 0000000000..4ce438fbb9 --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf @@ -0,0 +1,172 @@ +# Ensemble Stat Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# Output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h + +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_24h.conf + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PcpCombine, EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# Accumulation interval available in observation data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# Forecast data description variables +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB +FCST_IS_PROB = false + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = 0 +OBS_FILE_WINDOW_END = 0 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_24h_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = APCP +ENS_VAR1_LEVELS = A24 +ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A24 + +# FIGURE OUT WHAT BIN SIZE SHOULD BE +FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION +OBS_VAR1_NAME = {FCST_VAR1_NAME} +OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} + +OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + diff --git a/ush/templates/parm/metplus/EnsembleStat_REFC.conf b/ush/templates/parm/metplus/EnsembleStat_REFC.conf new file mode 100644 index 0000000000..393f6037bd --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_REFC.conf @@ -0,0 +1,119 @@ +# Ensemble Stat Grid to Grid REFC Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC + +[filename_templates] +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_REFC.conf + +[config] + +## Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} + +# Name to identify observation data in output +OBTYPE = MRMS + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = -300 +OBS_FILE_WINDOW_END = 300 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_REFC_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_REFC + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = REFC +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = ge20, ge30, ge40, ge50 + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = REFC +FCST_VAR1_LEVELS = L0 + +FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION + +OBS_VAR1_NAME = MergedReflectivityQCComposite +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + diff --git a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf new file mode 100644 index 0000000000..a90a9c7549 --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf @@ -0,0 +1,118 @@ +# Ensemble Stat Grid to Grid RETOP Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP + +[filename_templates] +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for grid observations. +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_RETOP.conf + +[config] + +## Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = EnsembleStat + +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times + +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT = 3600 + +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} + +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} + +# Name to identify observation data in output +OBTYPE = MRMS + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 +OBS_ENSEMBLE_STAT_WINDOW_END = 0 + +OBS_FILE_WINDOW_BEGIN = -300 +OBS_FILE_WINDOW_END = 300 + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_RETOP_{OBTYPE} + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_REFC + +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION +ENS_VAR1_NAME = RETOP +ENS_VAR1_LEVELS = L0 +ENS_VAR1_THRESH = ge20, ge30, ge40, ge50 +ENS_VAR1_OPTIONS = convert(x) = M_to_KFT(x); + +# Forecast Variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = RETOP +FCST_VAR1_LEVELS = L0 +FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + +# Observation Variables and levels as specified in the obs field dictionary +# of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION + +OBS_VAR1_NAME = EchoTop18 +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf new file mode 100644 index 0000000000..09d2572d72 --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -0,0 +1,170 @@ +# Ensemble Stat Grid to Point Precipitation Example + +[dir] +# Input and output directories for pb2nc +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +# Input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} + +# Point observation input dir for ensemble_stat (can also set grid obs) +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# Output directory for ensemble_stat +OUTPUT_BASE = {ENV[EXPTDIR]} +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc + +[filename_templates] + +# Input and output templates for pb2nc +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Input template for EnsembleStat can also be defined using a single +# member with wildcard characters to find multiple files +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.conus_surface.conf + +# Location of MET poly files +#ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PB2NC, EnsembleStat + +# Time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# List of forecast leads to process. +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# The MET ensemble_stat logging level +LOG_ENSEMBLE_STAT_VERBOSITY = 3 + +# MET Configuration files for pb2nc +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# if True, pb2nc will skip processing a file if the output already exists +# used to speed up runs and reduce redundancy +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# False for no time summary, True otherwise +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format +PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize + +# For both PB2NC and ensemble_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_ENSEMBLE_STAT_WINDOW_END = {OBS_WINDOW_END} + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = NONE + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_point + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]} +OBTYPE = NDAS + +# Used in the MET config file +ENSEMBLE_STAT_MESSAGE_TYPE = ADPSFC + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Variables and levels as specified in the field dictionary of the MET +# configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION + +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = Z02 +ENS_VAR1_THRESH = >=293, >=298, >=303 + +ENS_VAR2_NAME = DPT +ENS_VAR2_LEVELS = Z2 +ENS_VAR2_THRESH = >=288, >=293, >=298 + +ENS_VAR3_NAME = WIND +ENS_VAR3_LEVELS = Z10 +ENS_VAR3_THRESH = >=5, >=10 +ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind + +BOTH_VAR1_NAME = TMP +BOTH_VAR1_LEVELS = Z2 +BOTH_VAR1_THRESH = >=293, >=298, >=303 + +BOTH_VAR2_NAME = DPT +BOTH_VAR2_LEVELS = Z2 +BOTH_VAR2_THRESH = >=288, >=293, >=298 + +BOTH_VAR3_NAME = WIND +BOTH_VAR3_LEVELS = Z10 +BOTH_VAR3_THRESH = >=5, >=10 +BOTH_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf new file mode 100644 index 0000000000..19d982ecd4 --- /dev/null +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -0,0 +1,173 @@ +# Ensemble Stat Grid to Point Precipitation Example + +[dir] +# Input and output directories for pb2nc +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +# Input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} + +# Point observation input dir for ensemble_stat (can also set grid obs) +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# Output directory for ensemble_stat +OUTPUT_BASE = {ENV[EXPTDIR]} +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air + +[filename_templates] + +# Input and output templates for pb2nc +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Input ensemble template - comma separated list of ensemble members +#FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = +# mem01/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, +# mem02/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, + +# Input template for EnsembleStat can also be defined using a single +# member with wildcard characters to find multiple files +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.upper_air.conf + +# Location of MET poly files +#ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +[config] +# Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PB2NC, EnsembleStat + +# Time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# List of forecast leads to process. +LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# The MET ensemble_stat logging level +LOG_ENSEMBLE_STAT_VERBOSITY = 3 + +# MET Configuration files for pb2nc +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# if True, pb2nc will skip processing a file if the output already exists +# used to speed up runs and reduce redundancy +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# False for no time summary, True otherwise +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format +PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize + +# For both PB2NC and ensemble_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +OBS_ENSEMBLE_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_ENSEMBLE_STAT_WINDOW_END = {OBS_WINDOW_END} + +# number of expected members for ensemble. Should correspond with the +# number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE +ENSEMBLE_STAT_N_MEMBERS = 10 + +# ens.ens_thresh value in the MET config file +# threshold for ratio of valid files to expected files to allow app to run +ENSEMBLE_STAT_ENS_THRESH = 0.05 + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = NONE + +ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_point + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]} +OBTYPE = NDAS + +# Used in the MET config file +ENSEMBLE_STAT_MESSAGE_TYPE = ADPUPA + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +ENSEMBLE_STAT_MASK_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# MET_OBS_ERR_TABLE is not required. +# If the variable is not defined, or the value is not set +# than the MET default is used. +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Variables and levels as specified in the field dictionary of the MET +# configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION + +ENS_VAR1_NAME = TMP +ENS_VAR1_LEVELS = P850 +ENS_VAR1_THRESH = >=288, >=293, >=298 + +ENS_VAR2_NAME = DPT +ENS_VAR2_LEVELS = P850 +ENS_VAR2_THRESH = >=283, >=288, >=293 + +ENS_VAR3_NAME = WIND +ENS_VAR3_LEVELS = P850 +ENS_VAR3_THRESH = >=5, >=10 + +BOTH_VAR1_NAME = TMP +BOTH_VAR1_LEVELS = P850 +BOTH_VAR1_THRESH = >=288, >=293, >=298 + +BOTH_VAR2_NAME = DPT +BOTH_VAR2_LEVELS = P850 +BOTH_VAR2_THRESH = >=283, >=288, >=293 + +BOTH_VAR3_NAME = WIND +BOTH_VAR3_LEVELS = P850 +BOTH_VAR3_THRESH = >=5, >=10 + +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf new file mode 100644 index 0000000000..16cd58e7fb --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf @@ -0,0 +1,95 @@ +# Ensemble Mean Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP01.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_01h_{OBTYPE}_mean + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_01_A01_ENS_MEAN +FCST_VAR1_LEVELS = A01 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A01 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf new file mode 100644 index 0000000000..81b3d9a6d6 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf @@ -0,0 +1,129 @@ +# Ensemble Prob Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP01.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_01h_{OBTYPE}_prob + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +# +FCST_VAR1_NAME = APCP_01_A01_ENS_FREQ_gt0.0 +FCST_VAR1_LEVELS = A01 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A01 +OBS_VAR1_THRESH = >0.0 + +FCST_VAR2_NAME = APCP_01_A01_ENS_FREQ_ge0.254 +FCST_VAR2_LEVELS = A01 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = APCP +OBS_VAR2_LEVELS = A01 +OBS_VAR2_THRESH = >=0.254 + +FCST_VAR3_NAME = APCP_01_A01_ENS_FREQ_ge0.508 +FCST_VAR3_LEVELS = A01 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = APCP +OBS_VAR3_LEVELS = A01 +OBS_VAR3_THRESH = >=0.508 + +FCST_VAR4_NAME = APCP_01_A01_ENS_FREQ_ge1.27 +FCST_VAR4_LEVELS = A01 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = APCP +OBS_VAR4_LEVELS = A01 +OBS_VAR4_THRESH = >=1.27 + +FCST_VAR5_NAME = APCP_01_A01_ENS_FREQ_ge2.54 +FCST_VAR5_LEVELS = A01 +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = APCP +OBS_VAR5_LEVELS = A01 +OBS_VAR5_THRESH = >=2.54 + + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf new file mode 100644 index 0000000000..536ce241f4 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf @@ -0,0 +1,89 @@ +# Ensemble Mean Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP03.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_03h_{OBTYPE}_mean + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A3_ENS_MEAN +FCST_VAR1_LEVELS = A3 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A3 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf new file mode 100644 index 0000000000..984eba7f01 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf @@ -0,0 +1,138 @@ +# Ensemble Prob Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP03.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_03h_{OBTYPE}_prob + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A3_ENS_FREQ_gt0.0 +FCST_VAR1_LEVELS = A03 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A03 +OBS_VAR1_THRESH = >0.0 + +FCST_VAR2_NAME = APCP_A3_ENS_FREQ_ge0.254 +FCST_VAR2_LEVELS = A03 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = APCP +OBS_VAR2_LEVELS = A03 +OBS_VAR2_THRESH = >=0.254 + +FCST_VAR3_NAME = APCP_A3_ENS_FREQ_ge0.508 +FCST_VAR3_LEVELS = A03 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = APCP +OBS_VAR3_LEVELS = A03 +OBS_VAR3_THRESH = >=0.508 + +FCST_VAR4_NAME = APCP_A3_ENS_FREQ_ge1.27 +FCST_VAR4_LEVELS = A03 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = APCP +OBS_VAR4_LEVELS = A03 +OBS_VAR4_THRESH = >=1.27 + +FCST_VAR5_NAME = APCP_A3_ENS_FREQ_ge2.54 +FCST_VAR5_LEVELS = A03 +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = APCP +OBS_VAR5_LEVELS = A03 +OBS_VAR5_THRESH = >=2.54 + +FCST_VAR6_NAME = APCP_A3_ENS_FREQ_ge3.810 +FCST_VAR6_LEVELS = A03 +FCST_VAR6_THRESH = ==0.1 + +OBS_VAR6_NAME = APCP +OBS_VAR6_LEVELS = A03 +OBS_VAR6_THRESH = >=3.810 + +FCST_VAR7_NAME = APCP_A3_ENS_FREQ_ge6.350 +FCST_VAR7_LEVELS = A03 +FCST_VAR7_THRESH = ==0.1 + +OBS_VAR7_NAME = APCP +OBS_VAR7_LEVELS = A03 +OBS_VAR7_THRESH = >=6.350 + + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf new file mode 100644 index 0000000000..b10707f0fa --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf @@ -0,0 +1,89 @@ +# Ensemble Mean Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP06.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_06h_{OBTYPE}_mean + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A6_ENS_MEAN +FCST_VAR1_LEVELS = A06 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A06 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf new file mode 100644 index 0000000000..7c5e086721 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf @@ -0,0 +1,138 @@ +# Ensemble Prob Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP06.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_06h_{OBTYPE}_prob + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A6_ENS_FREQ_gt0.0 +FCST_VAR1_LEVELS = A06 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A06 +OBS_VAR1_THRESH = >0.0 + +FCST_VAR2_NAME = APCP_A6_ENS_FREQ_ge0.254 +FCST_VAR2_LEVELS = A06 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = APCP +OBS_VAR2_LEVELS = A06 +OBS_VAR2_THRESH = >=0.254 + +FCST_VAR3_NAME = APCP_A6_ENS_FREQ_ge0.508 +FCST_VAR3_LEVELS = A06 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = APCP +OBS_VAR3_LEVELS = A06 +OBS_VAR3_THRESH = >=0.508 + +FCST_VAR4_NAME = APCP_A6_ENS_FREQ_ge1.27 +FCST_VAR4_LEVELS = A06 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = APCP +OBS_VAR4_LEVELS = A06 +OBS_VAR4_THRESH = >=1.27 + +FCST_VAR5_NAME = APCP_A6_ENS_FREQ_ge2.54 +FCST_VAR5_LEVELS = A06 +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = APCP +OBS_VAR5_LEVELS = A06 +OBS_VAR5_THRESH = >=2.54 + +FCST_VAR6_NAME = APCP_A6_ENS_FREQ_ge3.810 +FCST_VAR6_LEVELS = A06 +FCST_VAR6_THRESH = ==0.1 + +OBS_VAR6_NAME = APCP +OBS_VAR6_LEVELS = A06 +OBS_VAR6_THRESH = >=3.810 + +FCST_VAR7_NAME = APCP_A6_ENS_FREQ_ge6.350 +FCST_VAR7_LEVELS = A06 +FCST_VAR7_THRESH = ==0.1 + +OBS_VAR7_NAME = APCP +OBS_VAR7_LEVELS = A06 +OBS_VAR7_THRESH = >=6.350 + + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf new file mode 100644 index 0000000000..aa0c53e210 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf @@ -0,0 +1,89 @@ +# Ensemble Mean Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP24.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_24h_{OBTYPE}_mean + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A24_ENS_MEAN +FCST_VAR1_LEVELS = A24 + +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A24 + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf new file mode 100644 index 0000000000..26d6e6368b --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf @@ -0,0 +1,138 @@ +# Ensemble Prob Grid to Grid Precipitation Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP24.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob + +# Set obtype to vx +OBTYPE = CCPA + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_24h_{OBTYPE}_prob + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +FCST_VAR1_NAME = APCP_A24_ENS_FREQ_gt0.0 +FCST_VAR1_LEVELS = A24 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = APCP +OBS_VAR1_LEVELS = A24 +OBS_VAR1_THRESH = >0.0 + +FCST_VAR2_NAME = APCP_A24_ENS_FREQ_ge0.254 +FCST_VAR2_LEVELS = A24 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = APCP +OBS_VAR2_LEVELS = A24 +OBS_VAR2_THRESH = >=0.254 + +FCST_VAR3_NAME = APCP_A24_ENS_FREQ_ge0.508 +FCST_VAR3_LEVELS = A24 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = APCP +OBS_VAR3_LEVELS = A24 +OBS_VAR3_THRESH = >=0.508 + +FCST_VAR4_NAME = APCP_A24_ENS_FREQ_ge1.27 +FCST_VAR4_LEVELS = A24 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = APCP +OBS_VAR4_LEVELS = A24 +OBS_VAR4_THRESH = >=1.27 + +FCST_VAR5_NAME = APCP_A24_ENS_FREQ_ge2.54 +FCST_VAR5_LEVELS = A24 +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = APCP +OBS_VAR5_LEVELS = A24 +OBS_VAR5_THRESH = >=2.54 + +FCST_VAR6_NAME = APCP_A24_ENS_FREQ_ge3.810 +FCST_VAR6_LEVELS = A24 +FCST_VAR6_THRESH = ==0.1 + +OBS_VAR6_NAME = APCP +OBS_VAR6_LEVELS = A24 +OBS_VAR6_THRESH = >=3.810 + +FCST_VAR7_NAME = APCP_A24_ENS_FREQ_ge6.350 +FCST_VAR7_LEVELS = A24 +FCST_VAR7_THRESH = ==0.1 + +OBS_VAR7_NAME = APCP +OBS_VAR7_LEVELS = A24 +OBS_VAR7_THRESH = >=6.350 + + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/GridStat_REFC_mean.conf b/ush/templates/parm/metplus/GridStat_REFC_mean.conf new file mode 100644 index 0000000000..098e2c0102 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_REFC_mean.conf @@ -0,0 +1,94 @@ +# Ensemble Mean Composite Reflectivity Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# ANLYS +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.REFC.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean + +# Set obtype to vx +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_mean + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# Forecast/Observation variable Information +FCST_VAR1_NAME = REFC_L0_ENS_MEAN +FCST_VAR1_LEVELS = L0 + +BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 + +OBS_VAR1_NAME = MergedReflectivityQCComposite +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# Neighborhood shape and width +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_REFC_prob.conf b/ush/templates/parm/metplus/GridStat_REFC_prob.conf new file mode 100644 index 0000000000..d6ad7452a2 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_REFC_prob.conf @@ -0,0 +1,122 @@ +# Ensemble Prob Grid to Grid Composite Reflectivity Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.REFC.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_prob + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +FCST_VAR1_NAME = REFC_L0_ENS_FREQ_ge20 +FCST_VAR1_LEVELS = L0 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = MergedReflectivityQCComposite +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_THRESH = >=20.0 +OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +FCST_VAR2_NAME = REFC_L0_ENS_FREQ_ge30 +FCST_VAR2_LEVELS = L0 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = MergedReflectivityQCComposite +OBS_VAR2_LEVELS = Z500 +OBS_VAR2_THRESH = >=30.0 +OBS_VAR2_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +FCST_VAR3_NAME = REFC_L0_ENS_FREQ_ge40 +FCST_VAR3_LEVELS = L0 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = MergedReflectivityQCComposite +OBS_VAR3_LEVELS = Z500 +OBS_VAR3_THRESH = >=40.0 +OBS_VAR3_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +FCST_VAR4_NAME = REFC_L0_ENS_FREQ_ge50 +FCST_VAR4_LEVELS = L0 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = MergedReflectivityQCComposite +OBS_VAR4_LEVELS = Z500 +OBS_VAR4_THRESH = >=50.0 +OBS_VAR4_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/GridStat_RETOP_mean.conf b/ush/templates/parm/metplus/GridStat_RETOP_mean.conf new file mode 100644 index 0000000000..6755ff2ee6 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_RETOP_mean.conf @@ -0,0 +1,95 @@ +# Ensemble Mean Composite RETOP Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_mean + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# ANLYS +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.RETOP.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +#LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_mean + +# Set obtype to vx +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_mean +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_mean + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# Forecast/Observation variable Information +FCST_VAR1_NAME = RETOP_L0_ENS_MEAN +FCST_VAR1_LEVELS = L0 +FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); + +BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 + +OBS_VAR1_NAME = EchoTop18 +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_OPTIONS = censor_thresh = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); + +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# Neighborhood shape and width +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# Forecast data description variables +FCST_IS_PROB = False + diff --git a/ush/templates/parm/metplus/GridStat_RETOP_prob.conf b/ush/templates/parm/metplus/GridStat_RETOP_prob.conf new file mode 100644 index 0000000000..41741a067b --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_RETOP_prob.conf @@ -0,0 +1,122 @@ +# Ensemble Prob Grid to Grid Composite RETOP Example + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {OUTPUT_BASE}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_prob + +[filename_templates] +# format of filenames +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Location of MET poly files +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.RETOP.conf + +[config] +# List of applications to run +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +# LEAD_SEQ = begin_end_incr(1,24,1) +LEAD_SEQ = {ENV[fhr_list]} + +# Options are times, processes +# times = run all items in the PROCESS_LIST for a single initialization +# time, then repeat until all times have been evaluated. +# processes = run each item in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST. +LOOP_ORDER = times + +# Model to verify +MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB + +# Set obtype to vx +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# location of grid_stat MET config file +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_prob +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_prob + +# run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Set grid to verify on +GRID_STAT_REGRID_TO_GRID = FCST + +# list of variables to compare +FCST_VAR1_NAME = RETOP_L0_ENS_FREQ_ge20 +FCST_VAR1_LEVELS = L0 +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = EchoTop18 +OBS_VAR1_LEVELS = Z500 +OBS_VAR1_THRESH = >=20.0 +OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); + +FCST_VAR2_NAME = RETOP_L0_ENS_FREQ_ge30 +FCST_VAR2_LEVELS = L0 +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = EchoTop18 +OBS_VAR2_LEVELS = Z500 +OBS_VAR2_THRESH = >=30.0 +OBS_VAR2_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); + +FCST_VAR3_NAME = RETOP_L0_ENS_FREQ_ge40 +FCST_VAR3_LEVELS = L0 +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = EchoTop18 +OBS_VAR3_LEVELS = Z500 +OBS_VAR3_THRESH = >=40.0 +OBS_VAR3_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); + +FCST_VAR4_NAME = RETOP_L0_ENS_FREQ_ge50 +FCST_VAR4_LEVELS = L0 +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = EchoTop18 +OBS_VAR4_LEVELS = Z500 +OBS_VAR4_THRESH = >=50.0 +OBS_VAR4_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); + +# Neighborhood shape and widths +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 1f9e3554f9..5fd3f58c9b 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -1,17 +1,21 @@ +# Grid to Point Example + [dir] # Input and output data directories PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +INPUT_BASE = {ENV[INPUT_BASE]} FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc [filename_templates] # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR @@ -23,7 +27,13 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat +POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.conus_surface.conf [config] @@ -73,7 +83,7 @@ PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND # For defining the time periods for summarization # False for no time summary, True otherwise @@ -135,26 +145,31 @@ BOTH_VAR5_NAME = VGRD BOTH_VAR5_LEVELS = Z10 BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts -BOTH_VAR6_NAME = PRMSL -BOTH_VAR6_LEVELS = Z0 +BOTH_VAR6_NAME = WIND +BOTH_VAR6_LEVELS = Z10 +BOTH_VAR6_THRESH = >=2.572 ;; m/s or 5kts +BOTH_VAR6_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind -BOTH_VAR7_NAME = TCDC -BOTH_VAR7_LEVELS = L0 -BOTH_VAR7_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } +BOTH_VAR7_NAME = PRMSL +BOTH_VAR7_LEVELS = Z0 -BOTH_VAR8_NAME = VIS +BOTH_VAR8_NAME = TCDC BOTH_VAR8_LEVELS = L0 -BOTH_VAR8_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 -BOTH_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } - -BOTH_VAR9_NAME = GUST -BOTH_VAR9_LEVELS = Z0 - -FCST_VAR10_NAME = HGT -FCST_VAR10_LEVELS = L0 -FCST_VAR10_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; -FCST_VAR10_THRESH = <152, <305, <914, <1520, <3040, >=914 -OBS_VAR10_NAME = CEILING -OBS_VAR10_LEVELS = L0 -OBS_VAR10_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } -OBS_VAR10_THRESH = <152, <305, <914, <1520, <3040, >=914 +BOTH_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +BOTH_VAR9_NAME = VIS +BOTH_VAR9_LEVELS = L0 +BOTH_VAR9_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 +BOTH_VAR9_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +BOTH_VAR10_NAME = GUST +BOTH_VAR10_LEVELS = Z0 + +FCST_VAR11_NAME = HGT +FCST_VAR11_LEVELS = L0 +FCST_VAR11_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; +FCST_VAR11_THRESH = <152, <305, <914, <1520, <3040, >=914 +OBS_VAR11_NAME = CEILING +OBS_VAR11_LEVELS = L0 +OBS_VAR11_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } +OBS_VAR11_THRESH = <152, <305, <914, <1520, <3040, >=914 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf new file mode 100644 index 0000000000..c5adc81df5 --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf @@ -0,0 +1,151 @@ +# Ensemble Mean Grid to Point Example + +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_mean + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.conus_surface.conf + +[config] +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_mean + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]}_mean +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPSFC_{OBTYPE}_mean + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPSFC + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = TMP_Z2_ENS_MEAN +FCST_VAR1_LEVELS = Z2 +FCST_VAR1_THRESH = >=293, >=298, >=303 + +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = Z2 +OBS_VAR1_THRESH = >=293, >=298, >=303 + +FCST_VAR2_NAME = DPT_Z2_ENS_MEAN +FCST_VAR2_LEVELS = Z2 +FCST_VAR2_THRESH = >=288, >=293, >=298 + +OBS_VAR2_NAME = DPT +OBS_VAR2_LEVELS = Z2 +OBS_VAR2_THRESH = >=288, >=293, >=298 + +FCST_VAR3_NAME = WIND_Z10_ENS_MEAN +FCST_VAR3_LEVELS = Z10 +FCST_VAR3_THRESH = >=5, >=10 + +OBS_VAR3_NAME = WIND +OBS_VAR3_LEVELS = Z10 +OBS_VAR3_THRESH = >=5, >=10 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf new file mode 100644 index 0000000000..f7ef100f77 --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -0,0 +1,195 @@ +# Ensemble Prob Grid to Point Example + +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_prob + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.conus_surface.conf + +[config] +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_prob + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]}_prob +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPSFC_{OBTYPE}_prob + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPSFC + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = TMP_Z2_ENS_FREQ_ge293 +FCST_VAR1_LEVELS = (*,*) +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = Z2 +OBS_VAR1_THRESH = >=293 + +FCST_VAR2_NAME = TMP_Z2_ENS_FREQ_ge298 +FCST_VAR2_LEVELS = (*,*) +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = TMP +OBS_VAR2_LEVELS = Z2 +OBS_VAR2_THRESH = >=298 + +FCST_VAR3_NAME = TMP_Z2_ENS_FREQ_ge303 +FCST_VAR3_LEVELS = (*,*) +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = TMP +OBS_VAR3_LEVELS = Z2 +OBS_VAR3_THRESH = >=303 + +FCST_VAR4_NAME = DPT_Z2_ENS_FREQ_ge288 +FCST_VAR4_LEVELS = (*,*) +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = DPT +OBS_VAR4_LEVELS = Z2 +OBS_VAR4_THRESH = >=288 + +FCST_VAR5_NAME = DPT_Z2_ENS_FREQ_ge293 +FCST_VAR5_LEVELS = (*,*) +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = DPT +OBS_VAR5_LEVELS = Z2 +OBS_VAR5_THRESH = >=293 + +FCST_VAR6_NAME = DPT_Z2_ENS_FREQ_ge298 +FCST_VAR6_LEVELS = (*,*) +FCST_VAR6_THRESH = ==0.1 + +OBS_VAR6_NAME = DPT +OBS_VAR6_LEVELS = Z2 +OBS_VAR6_THRESH = >=298 + +FCST_VAR7_NAME = WIND_Z10_ENS_FREQ_ge5 +FCST_VAR7_LEVELS = Z10 +FCST_VAR7_THRESH = ==0.1 + +OBS_VAR7_NAME = WIND +OBS_VAR7_LEVELS = Z10 +OBS_VAR7_THRESH = >=5 + +FCST_VAR8_NAME = WIND_Z10_ENS_FREQ_ge10 +FCST_VAR8_LEVELS = Z10 +FCST_VAR8_THRESH = ==0.1 + +OBS_VAR8_NAME = WIND +OBS_VAR8_LEVELS = Z10 +OBS_VAR8_THRESH = >=10 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index f9d3083caf..cd05d930b2 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -1,21 +1,26 @@ +# Grid to Point Example + [dir] # Input and output data directories PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +INPUT_BASE = {ENV[INPUT_BASE]} FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air [filename_templates] # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} + # Template to use to write output from PB2NC PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc @@ -23,10 +28,15 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/point_stat +POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat -[config] +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.upper_air.conf +[config] # List of applications to run PROCESS_LIST = PB2NC, PointStat @@ -73,7 +83,7 @@ PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND # For defining the time periods for summarization # False for no time summary, True otherwise @@ -135,30 +145,34 @@ BOTH_VAR5_NAME = VGRD BOTH_VAR5_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts -BOTH_VAR6_NAME = HGT -BOTH_VAR6_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 - -BOTH_VAR7_NAME = SPFH -BOTH_VAR7_LEVELS = P1000, P850, P700, P500, P400, P300 - -FCST_VAR8_NAME = CAPE -FCST_VAR8_LEVELS = L0 -FCST_VAR8_OPTIONS = cnt_thresh = [ >0 ]; -FCST_VAR8_THRESH = >500, >1000, >1500, >2000, >3000, >4000 -OBS_VAR8_NAME = CAPE -OBS_VAR8_LEVELS = L0-100000 -OBS_VAR8_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; -OBS_VAR8_THRESH = >500, >1000, >1500, >2000, >3000, >4000 - -FCST_VAR9_NAME = HPBL -FCST_VAR9_LEVELS = Z0 -OBS_VAR9_NAME = PBL -OBS_VAR9_LEVELS = L0 -OBS_VAR9_OPTIONS = desc = "TKE"; - -FCST_VAR10_NAME = HGT -FCST_VAR10_LEVELS = L0 -FCST_VAR10_OPTIONS = GRIB_lvl_typ = 220; +BOTH_VAR6_NAME = WIND +BOTH_VAR6_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 +BOTH_VAR6_THRESH = >=2.572 ;; m/s or 5kts + +BOTH_VAR7_NAME = HGT +BOTH_VAR7_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 + +BOTH_VAR8_NAME = SPFH +BOTH_VAR8_LEVELS = P1000, P850, P700, P500, P400, P300 + +FCST_VAR9_NAME = CAPE +FCST_VAR9_LEVELS = L0 +FCST_VAR9_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 +OBS_VAR9_NAME = CAPE +OBS_VAR9_LEVELS = L0-100000 +OBS_VAR9_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR9_THRESH = >500, >1000, >1500, >2000, >3000, >4000 + +FCST_VAR10_NAME = HPBL +FCST_VAR10_LEVELS = Z0 OBS_VAR10_NAME = PBL OBS_VAR10_LEVELS = L0 -OBS_VAR10_OPTIONS = desc = "RI"; +OBS_VAR10_OPTIONS = desc = "TKE"; + +FCST_VAR11_NAME = HGT +FCST_VAR11_LEVELS = L0 +FCST_VAR11_OPTIONS = GRIB_lvl_typ = 220; +OBS_VAR11_NAME = PBL +OBS_VAR11_LEVELS = L0 +OBS_VAR11_OPTIONS = desc = "RI"; diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf new file mode 100644 index 0000000000..e6a1a062ad --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -0,0 +1,152 @@ +# Ensemble Mean Grid to Point Example + +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_mean + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.upper_air.conf + +[config] +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) +#LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_mean + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]}_mean +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPUPA_{OBTYPE}_mean + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPUPA + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = TMP_P850_ENS_MEAN +FCST_VAR1_LEVELS = P850 +FCST_VAR1_THRESH = >=288, >=293, >=298 + +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = P850 +OBS_VAR1_THRESH = >=288, >=293, >=298 + +FCST_VAR2_NAME = DPT_P850_ENS_MEAN +FCST_VAR2_LEVELS = P850 +FCST_VAR2_THRESH = >=283, >=288, >=293 + +OBS_VAR2_NAME = DPT +OBS_VAR2_LEVELS = P850 +OBS_VAR2_THRESH = >=283, >=288, >=293 + +FCST_VAR3_NAME = WIND_P850_ENS_MEAN +FCST_VAR3_LEVELS = P850 +FCST_VAR3_THRESH = >=5, >=10 + +OBS_VAR3_NAME = WIND +OBS_VAR3_LEVELS = P850 +OBS_VAR3_THRESH = >=5, >=10 diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf new file mode 100644 index 0000000000..bb3790689d --- /dev/null +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -0,0 +1,196 @@ +# Ensemble Prob Grid to Point Example + +[dir] +# Input and output data directories +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} + +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_prob + +[filename_templates] +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.upper_air.conf + +[config] +# List of applications to run +PROCESS_LIST = PB2NC, PointStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +LOOP_BY = INIT + +# Start time for METplus run +INIT_BEG = {ENV[CDATE]} +# End time for METplus run +INIT_END = {ENV[CDATE]} +# Increment between METplus runs in seconds. Must be > 60 +INIT_INCREMENT = 3600 +INIT_TIME_FMT = %Y%m%d%H + +# list of forecast leads to process. +LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) +#LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +LOOP_ORDER = times + +# Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only +LOG_POINT_STAT_VERBOSITY = 2 + +# For both PB2NC and point_stat +OBS_WINDOW_BEGIN = -1799 +OBS_WINDOW_END = 1800 + +PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +PB2NC_WINDOW_END = {OBS_WINDOW_END} + +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig + +# If set to True, skip run if the output file determined by the output directory and +# filename template already exists +PB2NC_SKIP_IF_OUTPUT_EXISTS = True + +# Values to pass to pb2nc config file using environment variables of the same name. +PB2NC_GRID = +PB2NC_POLY = +PB2NC_STATION_ID = +PB2NC_MESSAGE_TYPE = + +# Leave empty to process all +# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND + +# For defining the time periods for summarization +# False for no time summary, True otherwise +# The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False +PB2NC_TIME_SUMMARY_FLAG = False +PB2NC_TIME_SUMMARY_BEG = 000000 +PB2NC_TIME_SUMMARY_END = 235959 +PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC +PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 + +# Location of MET config file to pass to GridStat +# References PARM_BASE which is the location of the parm directory corresponding +# to the ush directory of the master_metplus.py script that is called +# or the value of the environment variable METPLUS_PARM_BASE if set +POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_prob + +# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. +MODEL = {ENV[MODEL]}_prob +OBTYPE = NDAS + +# Regrid to specified grid. Indicate NONE if no regridding, or the grid id +POINT_STAT_REGRID_TO_GRID = NONE + +POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPUPA_{OBTYPE}_prob + +# Verification Masking regions +# Indicate which grid and polygon masking region, if applicable +POINT_STAT_GRID = + +# List of full path to poly masking files. NOTE: Only short lists of poly +# files work (those that fit on one line), a long list will result in an +# environment variable that is too long, resulting in an error. For long +# lists of poly masking files (i.e. all the mask files in the NCEP_mask +# directory), define these in the MET point_stat configuration file. +POINT_STAT_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +POINT_STAT_STATION_ID = + +# Message types, if all message types are to be returned, leave this empty, +# otherwise indicate the message types of interest. +POINT_STAT_MESSAGE_TYPE = ADPUPA + +# Variables and levels as specified in the field dictionary of the MET +# point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION +FCST_VAR1_NAME = TMP_P850_ENS_FREQ_ge288 +FCST_VAR1_LEVELS = (*,*) +FCST_VAR1_THRESH = ==0.1 + +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = P850 +OBS_VAR1_THRESH = >=288 + +FCST_VAR2_NAME = TMP_P850_ENS_FREQ_ge293 +FCST_VAR2_LEVELS = (*,*) +FCST_VAR2_THRESH = ==0.1 + +OBS_VAR2_NAME = TMP +OBS_VAR2_LEVELS = P850 +OBS_VAR2_THRESH = >=293 + +FCST_VAR3_NAME = TMP_P850_ENS_FREQ_ge298 +FCST_VAR3_LEVELS = (*,*) +FCST_VAR3_THRESH = ==0.1 + +OBS_VAR3_NAME = TMP +OBS_VAR3_LEVELS = P850 +OBS_VAR3_THRESH = >=298 + +FCST_VAR4_NAME = DPT_P850_ENS_FREQ_ge283 +FCST_VAR4_LEVELS = (*,*) +FCST_VAR4_THRESH = ==0.1 + +OBS_VAR4_NAME = DPT +OBS_VAR4_LEVELS = P850 +OBS_VAR4_THRESH = >=283 + +FCST_VAR5_NAME = DPT_P850_ENS_FREQ_ge288 +FCST_VAR5_LEVELS = (*,*) +FCST_VAR5_THRESH = ==0.1 + +OBS_VAR5_NAME = DPT +OBS_VAR5_LEVELS = P850 +OBS_VAR5_THRESH = >=288 + +FCST_VAR6_NAME = DPT_P850_ENS_FREQ_ge293 +FCST_VAR6_LEVELS = (*,*) +FCST_VAR6_THRESH = ==0.1 + +OBS_VAR6_NAME = DPT +OBS_VAR6_LEVELS = P850 +OBS_VAR6_THRESH = >=293 + +FCST_VAR7_NAME = WIND_P850_ENS_FREQ_ge5 +FCST_VAR7_LEVELS = (*,*) +FCST_VAR7_THRESH = ==0.1 + +OBS_VAR7_NAME = WIND +OBS_VAR7_LEVELS = P850 +OBS_VAR7_THRESH = >=5 + +FCST_VAR8_NAME = WIND_P850_ENS_FREQ_ge10 +FCST_VAR8_LEVELS = (*,*) +FCST_VAR8_THRESH = ==0.1 + +OBS_VAR8_NAME = WIND +OBS_VAR8_LEVELS = P850 +OBS_VAR8_THRESH = >=10 + +# Forecast data description variables +FCST_IS_PROB = True +FCST_PROB_IN_GRIB_PDS = False diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf index 6aa8d198f3..9a3fdb2d1c 100644 --- a/ush/templates/parm/metplus/REFC.conf +++ b/ush/templates/parm/metplus/REFC.conf @@ -1,17 +1,19 @@ -# Composite Reflectivity Verification +# Composite Reflectivity Verification Options [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +STAGING_DIR = {OUTPUT_BASE}/stage/REFC + [filename_templates] # format of filenames FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 @@ -19,11 +21,17 @@ FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.REFC.conf + [config] # Model to verify MODEL = {ENV[MODEL]} @@ -37,7 +45,7 @@ OBS_NATIVE_DATA_TYPE = GRIB PROCESS_LIST = GridStat # location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFL +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/ush/templates/parm/metplus/RETOP.conf b/ush/templates/parm/metplus/RETOP.conf index 5b57d3c34c..5508bd600c 100644 --- a/ush/templates/parm/metplus/RETOP.conf +++ b/ush/templates/parm/metplus/RETOP.conf @@ -1,17 +1,19 @@ -# Composite Reflectivity Verification +# Echo Top Verification Options [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]}/postprd +INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {OUTPUT_BASE}/log +LOG_DIR = {ENV[EXPTDIR]}/log OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP + [filename_templates] # format of filenames FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 @@ -19,11 +21,17 @@ FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH # ANLYS OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/grid_stat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Location of MET poly files GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.RETOP.conf + [config] # Model to verify MODEL = {ENV[MODEL]} @@ -37,7 +45,7 @@ OBS_NATIVE_DATA_TYPE = GRIB PROCESS_LIST = GridStat # location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFL +GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} # time looping - options are INIT, VALID, RETRO, and REALTIME diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 96998bc50b..d5eb39b9b7 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -52,9 +52,11 @@ valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "n valid_vals_RUN_TASK_MAKE_OROG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_MAKE_SFC_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_RUN_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_WRITE_DOPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_VX_GRIDSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_TASK_VX_POINTSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_WRITE_DOPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_RUN_TASK_VX_ENSGRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_RUN_TASK_VX_ENSPOINT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From a2234624eb0a0a37b989ecd5b92ba795e28f10de Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Tue, 21 Sep 2021 21:32:39 -0600 Subject: [PATCH 087/203] Update develop branch for generic linux, MacOS capability (#539) ## DESCRIPTION OF CHANGES: This change will add the capability to run regional_workflow (as part of the SRW app) on MacOS and generic LINUX platforms. Most of these changes are identical to those in #402 (hash e6b8c74) but some additional modifications needed to be made due to intervening changes in the develop branch. ## TESTS CONDUCTED: Ran Graduate Student Test on new platforms: - my personal Mac machine (MacOS Catalina 10.15.7) MacOS with gnu 9.4.0 compilers. - Cheyenne compute node as a faux "stand-alone" machine, intel 19.1.1 compilers Ran suite of end-to-end tests on Cheyenne (intel/19.1.1) and Hera (intel/18.0.5.274). All passed as expected. Tests also passed on WCOSS, MacOS Mojave, RedHat Linux. ## ISSUE: Will resolve #369 --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 2 +- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_MAKE_GRID | 2 +- jobs/JREGIONAL_MAKE_ICS | 2 +- jobs/JREGIONAL_MAKE_LBCS | 2 +- jobs/JREGIONAL_MAKE_OROG | 2 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 2 +- jobs/JREGIONAL_RUN_FCST | 2 +- jobs/JREGIONAL_RUN_POST | 4 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- scripts/exregional_get_ccpa_files.sh | 8 +- scripts/exregional_get_extrn_mdl_files.sh | 6 +- scripts/exregional_get_mrms_files.sh | 6 +- scripts/exregional_get_ndas_files.sh | 14 +- scripts/exregional_make_grid.sh | 21 +- scripts/exregional_make_ics.sh | 16 +- scripts/exregional_make_lbcs.sh | 18 +- scripts/exregional_make_orog.sh | 36 +-- scripts/exregional_make_sfc_climo.sh | 13 +- scripts/exregional_run_ensgridvx.sh | 2 +- scripts/exregional_run_ensgridvx_mean.sh | 2 +- scripts/exregional_run_ensgridvx_prob.sh | 2 +- scripts/exregional_run_enspointvx.sh | 2 +- scripts/exregional_run_enspointvx_mean.sh | 2 +- scripts/exregional_run_enspointvx_prob.sh | 2 +- scripts/exregional_run_fcst.sh | 23 +- scripts/exregional_run_gridstatvx.sh | 4 +- scripts/exregional_run_pointstatvx.sh | 4 +- scripts/exregional_run_post.sh | 21 +- ush/bash_utils/change_case.sh | 239 ++++++++++++++++++ ush/bash_utils/check_for_preexist_dir_file.sh | 2 +- ush/bash_utils/check_var_valid_value.sh | 2 +- ush/bash_utils/count_files.sh | 2 +- ush/bash_utils/create_symlink_to_file.sh | 2 +- ush/bash_utils/define_macos_utilities.sh | 44 ++++ ush/bash_utils/filesys_cmds_vrfy.sh | 6 +- ush/bash_utils/get_charvar_from_netcdf.sh | 4 +- ush/bash_utils/get_elem_inds.sh | 4 +- .../get_manage_externals_config_property.sh | 6 +- ush/bash_utils/interpol_to_arbit_CRES.sh | 2 +- ush/bash_utils/is_array.sh | 2 +- ush/bash_utils/is_element_of.sh | 2 +- ush/bash_utils/print_input_args.sh | 6 +- ush/bash_utils/print_msg.sh | 8 +- ush/bash_utils/process_args.sh | 4 +- ush/bash_utils/save_restore_shell_opts.sh | 2 +- ush/bash_utils/set_bash_param.sh | 4 +- ush/bash_utils/set_file_param.sh | 4 +- ush/check_ruc_lsm.sh | 4 +- ush/cmp_expt_to_baseline.sh | 2 +- ush/compare_config_scripts.sh | 8 +- ush/config_defaults.sh | 30 +++ ush/create_diag_table_file.sh | 2 +- ush/create_model_configure_file.sh | 10 +- ush/generate_FV3LAM_wflow.sh | 86 ++++--- ush/get_extrn_mdl_file_dir_info.sh | 69 +++-- ush/launch_FV3LAM_wflow.sh | 16 +- ush/link_fix.sh | 4 +- ush/load_modules_run_task.sh | 87 +------ ush/make_grid_mosaic_file.sh | 2 +- ush/set_FV3nml_sfc_climo_filenames.sh | 6 +- ush/set_FV3nml_stoch_params.sh | 2 +- ush/set_cycle_dates.sh | 4 +- ush/set_extrn_mdl_params.sh | 2 +- ush/set_gridparams_ESGgrid.sh | 2 +- ush/set_gridparams_GFDLgrid.sh | 6 +- ush/set_ozone_param.sh | 8 +- ush/set_predef_grid_params.sh | 2 +- ush/set_thompson_mp_fix_files.sh | 4 +- ush/setup.sh | 124 +++++---- ush/source_util_funcs.sh | 24 +- ush/valid_param_vals.sh | 3 +- 81 files changed, 730 insertions(+), 364 deletions(-) create mode 100644 ush/bash_utils/change_case.sh create mode 100644 ush/bash_utils/define_macos_utilities.sh diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 6b373895e8..bfe2d7bcea 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -57,7 +57,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index 6f541ccd25..e63010d0d6 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -39,7 +39,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index 4ae197feed..dfe1f68a6c 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -35,7 +35,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 32c1d819e7..9514931cc1 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -35,7 +35,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index 641196137b..bbf22e0241 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -125,7 +125,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 01027b4ac9..9c5125f418 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index b46d0e950c..bc7afbaf71 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index e3935037d7..50b6c03697 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index f2993bbaff..24af27bfa3 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index a4806c765e..dbd4c80c4b 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -37,7 +37,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 68dcf95bd4..ed469dbe28 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -36,7 +36,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -113,7 +113,7 @@ cd_vrfy "${tmp_dir}" # #----------------------------------------------------------------------- # -fhr=$( printf "%s" "${fhr}" | sed -n -r -e "s/^([0-9]+)$/\1/p" ) +fhr=$( printf "%s" "${fhr}" | $SED -n -r -e "s/^([0-9]+)$/\1/p" ) if [ -z "$fhr" ]; then print_err_msg_exit "\ The forecast hour (fhr) must be a non-empty string consisting of only diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index 9fa88c2f7d..641df6f663 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -36,7 +36,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index 67945ca7b9..d8c4dcea04 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -36,7 +36,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index f792d4b77f..7ae543314b 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -36,7 +36,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index dac8873a2a..c240d7abeb 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -34,7 +34,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index 3ae1cb56dc..376d07b99f 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -34,7 +34,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 8810d0bcbc..1a47cfd124 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -34,7 +34,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index 300c79f9c6..b17fd29605 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -36,7 +36,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index f67cb55479..f995045aef 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -34,7 +34,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_get_ccpa_files.sh b/scripts/exregional_get_ccpa_files.sh index 3c1504292d..b0ab11f5bb 100755 --- a/scripts/exregional_get_ccpa_files.sh +++ b/scripts/exregional_get_ccpa_files.sh @@ -47,9 +47,9 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time - init_ut=`date -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time + init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time - vdate=`date -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time @@ -60,7 +60,7 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do # Calculate valid date - 1 day vdate_ut_m1=`expr ${vdate_ut} - 86400` - vdate_m1=`date -ud '1970-01-01 UTC '${vdate_ut_m1}' seconds' +%Y%m%d%H` + vdate_m1=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m1}' seconds' +%Y%m%d%H` vyyyymmdd_m1=`echo ${vdate_m1} | cut -c1-8` vyyyy_m1=`echo ${vdate_m1} | cut -c1-4` vmm_m1=`echo ${vdate_m1} | cut -c5-6` @@ -69,7 +69,7 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do # Calculate valid date + 1 day vdate_ut_p1=`expr ${vdate_ut} + 86400` - vdate_p1=`date -ud '1970-01-01 UTC '${vdate_ut_p1}' seconds' +%Y%m%d%H` + vdate_p1=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_p1}' seconds' +%Y%m%d%H` vyyyymmdd_p1=`echo ${vdate_p1} | cut -c1-8` vyyyy_p1=`echo ${vdate_p1} | cut -c1-4` vmm_p1=`echo ${vdate_p1} | cut -c5-6` diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 35e8d373bc..ac5127eb8b 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -524,7 +524,7 @@ not happen." # to the current directory. Then move the files. # rel_dir=$( printf "%s" "${extrn_mdl_arcvrel_dir}" | \ - sed -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) + $SED -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) mv_vrfy ${rel_dir}/* . # # Get the first subdirectory in rel_dir, i.e. the subdirectory before the @@ -533,7 +533,7 @@ not happen." # it. # subdir_to_remove=$( printf "%s" "${rel_dir}" | \ - sed -r 's%^([^/]*)(.*)%\1%' ) + $SED -r 's%^([^/]*)(.*)%\1%' ) rm_vrfy -rf ./${subdir_to_remove} # # If extrn_mdl_arcvrel_dir does not start with a "/" (and it is not diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index 8819959ecc..d0abf8f9bb 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -38,7 +38,7 @@ s_yyyy=`echo ${start_valid} | cut -c1-4` # year (YYYY) of start time s_mm=`echo ${start_valid} | cut -c5-6` # month (MM) of start time s_dd=`echo ${start_valid} | cut -c7-8` # day (DD) of start time s_hh=`echo ${start_valid} | cut -c9-10` # hour (HH) of start time -start_valid_ut=`date -ud ''${s_yyyy}-${s_mm}-${s_dd}' UTC '${s_hh}':00:00' +%s` # convert start time to universal time +start_valid_ut=`$DATE_UTIL -ud ''${s_yyyy}-${s_mm}-${s_dd}' UTC '${s_hh}':00:00' +%s` # convert start time to universal time end_fcst_sec=`expr ${fcst_length} \* 3600` # convert last forecast lead hour to seconds end_valid_ut=`expr ${start_valid_ut} + ${end_fcst_sec}` # calculate current forecast time in universal time @@ -48,7 +48,7 @@ current_fcst=0 fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds while [[ ${cur_ut} -le ${end_valid_ut} ]]; do - cur_time=`date -ud '1970-01-01 UTC '${cur_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + cur_time=`$DATE_UTIL -ud '1970-01-01 UTC '${cur_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time echo "cur_time=${cur_time}" # Calculate valid date info @@ -57,7 +57,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do vdd=`echo ${cur_time} | cut -c7-8` # day (DD) of time vhh=`echo ${cur_time} | cut -c9-10` # hour (HH) of time vyyyymmdd=`echo ${cur_time} | cut -c1-8` # YYYYMMDD of time - vinit_ut=`date -ud ''${vyyyy}-${vmm}-${vdd}' UTC '${vhh}':00:00' +%s` # convert time to universal time + vinit_ut=`$DATE_UTIL -ud ''${vyyyy}-${vmm}-${vdd}' UTC '${vhh}':00:00' +%s` # convert time to universal time # Create necessary raw and proc directories if [[ ! -d "$mrms_raw/${vyyyymmdd}" ]]; then diff --git a/scripts/exregional_get_ndas_files.sh b/scripts/exregional_get_ndas_files.sh index e1ec7f8d68..0105f2edb0 100755 --- a/scripts/exregional_get_ndas_files.sh +++ b/scripts/exregional_get_ndas_files.sh @@ -40,9 +40,9 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time - init_ut=`date -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time + init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time - vdate=`date -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time @@ -53,7 +53,7 @@ echo "yyyy mm dd hh= $yyyy $mm $dd $hh" echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" vdate_ut_m1h=`expr ${vdate_ut} - 3600` # calculate current forecast time in universal time - vdate_m1h=`date -ud '1970-01-01 UTC '${vdate_ut_m1h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate_m1h=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m1h}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd_m1h=`echo ${vdate_m1h} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy_m1h=`echo ${vdate_m1h} | cut -c1-4` # year (YYYY) of valid time vmm_m1h=`echo ${vdate_m1h} | cut -c5-6` # month (MM) of valid time @@ -61,7 +61,7 @@ echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" vhh_m1h=`echo ${vdate_m1h} | cut -c9-10` # forecast hour (HH) vdate_ut_m2h=`expr ${vdate_ut} - 7200` # calculate current forecast time in universal time - vdate_m2h=`date -ud '1970-01-01 UTC '${vdate_ut_m2h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate_m2h=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m2h}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd_m2h=`echo ${vdate_m2h} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy_m2h=`echo ${vdate_m2h} | cut -c1-4` # year (YYYY) of valid time vmm_m2h=`echo ${vdate_m2h} | cut -c5-6` # month (MM) of valid time @@ -69,7 +69,7 @@ echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" vhh_m2h=`echo ${vdate_m2h} | cut -c9-10` # forecast hour (HH) vdate_ut_m3h=`expr ${vdate_ut} - 10800` # calculate current forecast time in universal time - vdate_m3h=`date -ud '1970-01-01 UTC '${vdate_ut_m3h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate_m3h=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m3h}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd_m3h=`echo ${vdate_m3h} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy_m3h=`echo ${vdate_m3h} | cut -c1-4` # year (YYYY) of valid time vmm_m3h=`echo ${vdate_m3h} | cut -c5-6` # month (MM) of valid time @@ -77,7 +77,7 @@ echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" vhh_m3h=`echo ${vdate_m3h} | cut -c9-10` # forecast hour (HH) vdate_ut_m4h=`expr ${vdate_ut} - 14400` # calculate current forecast time in universal time - vdate_m4h=`date -ud '1970-01-01 UTC '${vdate_ut_m4h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate_m4h=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m4h}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd_m4h=`echo ${vdate_m4h} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy_m4h=`echo ${vdate_m4h} | cut -c1-4` # year (YYYY) of valid time vmm_m4h=`echo ${vdate_m4h} | cut -c5-6` # month (MM) of valid time @@ -85,7 +85,7 @@ echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" vhh_m4h=`echo ${vdate_m4h} | cut -c9-10` # forecast hour (HH) vdate_ut_m5h=`expr ${vdate_ut} - 18000` # calculate current forecast time in universal time - vdate_m5h=`date -ud '1970-01-01 UTC '${vdate_ut_m5h}' seconds' +%Y%m%d%H` # convert universal time to standard time + vdate_m5h=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m5h}' seconds' +%Y%m%d%H` # convert universal time to standard time vyyyymmdd_m5h=`echo ${vdate_m5h} | cut -c1-8` # forecast time (YYYYMMDD) vyyyy_m5h=`echo ${vdate_m5h} | cut -c1-4` # year (YYYY) of valid time vmm_m5h=`echo ${vdate_m5h} | cut -c5-6` # month (MM) of valid time diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 5d9bddaf66..9d4a1d9f6d 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -37,7 +37,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -135,11 +135,18 @@ case "$MACHINE" in ulimit -a ;; + "MACOS") + APRUN=time + ;; + + "LINUX") + APRUN=time + ;; + *) print_err_msg_exit "\ Run command has not been specified for this machine: - MACHINE = \"$MACHINE\" - APRUN = \"$APRUN\"" + MACHINE = \"$MACHINE\"" ;; esac @@ -422,6 +429,14 @@ uniform cubed-sphere grid equivalent resolution returned with nonzero exit code: exec_fp = \"${exec_fp}\"" +# Make sure 'ncdump' is available before we try to use it +if ! command -v ncdump &> /dev/null +then + print_err_msg_exit "\ +The utility 'ncdump' was not found in the environment. Be sure to add the +netCDF 'bin/' directory to your PATH." +fi + # Make the following (reading of res_equiv) a function in another file # so that it can be used both here and in the exregional_make_orog.sh # script. diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 41b00ac637..ec0fee623c 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -126,6 +126,20 @@ case "$MACHINE" in APRUN="ibrun" ;; + "MACOS") + APRUN=$RUN_CMD_UTILS + ;; + + "LINUX") + APRUN=$RUN_CMD_UTILS + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\"" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 3865f092a1..1b1f84ff88 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -126,6 +126,20 @@ case "$MACHINE" in APRUN="ibrun" ;; + "MACOS") + APRUN=$RUN_CMD_UTILS + ;; + + "LINUX") + APRUN=$RUN_CMD_UTILS + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\"" + ;; + esac # #----------------------------------------------------------------------- @@ -438,7 +452,7 @@ list file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): dd="${EXTRN_MDL_CDATE:6:2}" hh="${EXTRN_MDL_CDATE:8:2}" - cdate_crnt_fhr=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) + cdate_crnt_fhr=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) # # Get the month, day, and hour corresponding to the current forecast time # of the the external model. diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index ed7d43a0c0..e90b3b762c 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -35,7 +35,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -147,11 +147,18 @@ case "$MACHINE" in APRUN="time" ;; + "MACOS") + APRUN=time + ;; + + "LINUX") + APRUN=time + ;; + *) print_err_msg_exit "\ Run command has not been specified for this machine: - MACHINE = \"$MACHINE\" - APRUN = \"$APRUN\"" + MACHINE = \"$MACHINE\"" ;; esac @@ -226,7 +233,8 @@ cp_vrfy ${TOPO_DIR}/gmted2010.30sec.int fort.235 mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" mosaic_fp="$FIXLAM/${mosaic_fn}" -grid_fn=$( get_charvar_from_netcdf "${mosaic_fp}" "gridfiles" ) +grid_fn=$( get_charvar_from_netcdf "${mosaic_fp}" "gridfiles" ) || print_err_msg_exit "\ + get_charvar_from_netcdf function failed." grid_fp="${FIXLAM}/${grid_fn}" # #----------------------------------------------------------------------- @@ -284,7 +292,7 @@ cat "${input_redirect_fn}" # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " +print_info_msg "$VERBOSE" "\ Starting orography file generation..." $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ @@ -327,18 +335,17 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then cd_vrfy ${tmp_dir} mosaic_fn_gwd="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" mosaic_fp_gwd="$FIXLAM/${mosaic_fn_gwd}" - grid_fn_gwd=$( get_charvar_from_netcdf "${mosaic_fp_gwd}" "gridfiles" ) + grid_fn_gwd=$( get_charvar_from_netcdf "${mosaic_fp_gwd}" "gridfiles" ) || \ + print_err_msg_exit "get_charvar_from_netcdf function failed." grid_fp_gwd="${FIXLAM}/${grid_fn_gwd}" ls_fn="geo_em.d01.lat-lon.2.5m.HGT_M.nc" ss_fn="HGT.Beljaars_filtered.lat-lon.30s_res.nc" - if [ "${MACHINE}" = "WCOSS_CRAY" ]; then - relative_or_null="" - else - relative_or_null="--relative" - fi - ln_vrfy -fs ${relative_or_null} "${grid_fp_gwd}" "${tmp_dir}/${grid_fn_gwd}" - ln_vrfy -fs ${relative_or_null} "${FIXam}/${ls_fn}" "${tmp_dir}/${ls_fn}" - ln_vrfy -fs ${relative_or_null} "${FIXam}/${ss_fn}" "${tmp_dir}/${ss_fn}" + create_symlink_to_file target="${grid_fp_gwd}" symlink="${tmp_dir}/${grid_fn_gwd}" \ + relative="TRUE" + create_symlink_to_file target="${FIXam}/${ls_fn}" symlink="${tmp_dir}/${ls_fn}" \ + relative="TRUE" + create_symlink_to_file target="${FIXam}/${ss_fn}" symlink="${tmp_dir}/${ss_fn}" \ + relative="TRUE" input_redirect_fn="grid_info.dat" cat > "${input_redirect_fn}" < /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Get information about the script or function that calls this function. +# Note that caller_name will be set as follows: +# +# 1) If the caller is a function, caller_name will be set to the name of +# that function. +# 2) If the caller is a sourced script, caller_name will be set to +# "script". Note that a sourced script cannot be the top level +# script since by defintion, it is sourced by another script or func- +# tion. +# 3) If the caller is the top-level script, caller_name will be set to +# "main". +# +# Thus, if caller_name is set to "script" or "main", the caller is a +# script, and if it is set to anything else, the caller is a function. +# +#----------------------------------------------------------------------- +# + local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) + local caller_fn=$( basename "${caller_fp}" ) + local caller_dir=$( dirname "${caller_fp}" ) + local caller_name="${FUNCNAME[1]}" +# +# Get input string + + local input + + if [ "$#" -eq 1 ]; then + + input="$1" + +# +#----------------------------------------------------------------------- +# +# If no arguments or more than one, print out a usage message and exit. +# +#----------------------------------------------------------------------- +# + else + + print_err_msg_exit " +Incorrect number of arguments specified: + + Function name: \"${func_name}\" + Number of arguments specified: $# + +Usage: + + ${func_name} string + +where: + + string: + This is the string that should be converted to uppercase and echoed. +" + + fi + +# Echo the input string as upperercase + +echo $input| tr '[a-z]' '[A-Z]' + +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + + +} + + +# +#----------------------------------------------------------------------- +# +# Function to echo the given string as a lowercase string +# +#----------------------------------------------------------------------- +# +function echo_lowercase() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Get information about the script or function that calls this function. +# Note that caller_name will be set as follows: +# +# 1) If the caller is a function, caller_name will be set to the name of +# that function. +# 2) If the caller is a sourced script, caller_name will be set to +# "script". Note that a sourced script cannot be the top level +# script since by defintion, it is sourced by another script or func- +# tion. +# 3) If the caller is the top-level script, caller_name will be set to +# "main". +# +# Thus, if caller_name is set to "script" or "main", the caller is a +# script, and if it is set to anything else, the caller is a function. +# +#----------------------------------------------------------------------- +# + local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) + local caller_fn=$( basename "${caller_fp}" ) + local caller_dir=$( dirname "${caller_fp}" ) + local caller_name="${FUNCNAME[1]}" +# +# Get input string + + local input + + if [ "$#" -eq 1 ]; then + + input="$1" + +# +#----------------------------------------------------------------------- +# +# If no arguments or more than one, print out a usage message and exit. +# +#----------------------------------------------------------------------- +# + else + + print_err_msg_exit " +Incorrect number of arguments specified: + + Function name: \"${func_name}\" + Number of arguments specified: $# + +Usage: + + ${func_name} string + +where: + + string: + This is the string that should be converted to lowercase and echoed. +" + + fi + +# Echo the input string as lowercase + +echo $input| tr '[A-Z]' '[a-z]' + +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + + +} + diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh index 7b9b597722..dac2688a31 100644 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ b/ush/bash_utils/check_for_preexist_dir_file.sh @@ -26,7 +26,7 @@ function check_for_preexist_dir_file() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 723460ea14..7a0e20707b 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -25,7 +25,7 @@ function check_var_valid_value() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/count_files.sh b/ush/bash_utils/count_files.sh index 633bbf7ce4..c80f342cb1 100644 --- a/ush/bash_utils/count_files.sh +++ b/ush/bash_utils/count_files.sh @@ -25,7 +25,7 @@ function count_files() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index e54bf0a36a..c14a35bdc6 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -31,7 +31,7 @@ function create_symlink_to_file() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/define_macos_utilities.sh b/ush/bash_utils/define_macos_utilities.sh new file mode 100644 index 0000000000..d89780944d --- /dev/null +++ b/ush/bash_utils/define_macos_utilities.sh @@ -0,0 +1,44 @@ +# +#----------------------------------------------------------------------- +# +# This script defines MacOS-specific UNIX command-line utilities that +# mimic the functionality of the GNU equivalents. +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# +# Check if we are on a Darwin machine; if so we need to use the gnu-like +# equivalent of readlink and sed. +# +#----------------------------------------------------------------------- +# +darwinerror () { + + utility=$1 + echo >&2 " +For Darwin-based operating systems (MacOS), the '${utility}' utility is required to run the UFS SRW Application. +Reference the User's Guide for more information about platform requirements. +Aborting. +" + exit 1 +} + + if [[ $(uname -s) == Darwin ]]; then + export READLINK=greadlink + command -v $READLINK >/dev/null 2>&1 || darwinerror $READLINK + export SED=gsed + command -v $SED >/dev/null 2>&1 || darwinerror $SED + export DATE_UTIL=gdate + command -v $DATE_UTIL >/dev/null 2>&1 || darwinerror $DATE_UTIL + export LN_UTIL=gln + command -v $LN_UTIL >/dev/null 2>&1 || darwinerror $LN_UTIL + else + export READLINK=readlink + export SED=sed + export DATE_UTIL=date + export LN_UTIL=ln + fi + diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index 57a0870580..d147fc1079 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -28,7 +28,7 @@ function filesys_cmd_vrfy() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -65,7 +65,7 @@ function filesys_cmd_vrfy() { # #----------------------------------------------------------------------- # - local caller_fp=$( readlink -f "${BASH_SOURCE[2]}" ) + local caller_fp=$( $READLINK -f "${BASH_SOURCE[2]}" ) local caller_fn=$( basename "${caller_fp}" ) local caller_dir=$( dirname "${caller_fp}" ) local caller_name="${FUNCNAME[2]}" @@ -258,7 +258,7 @@ function rm_vrfy() { function ln_vrfy() { { save_shell_opts; set -u +x; } > /dev/null 2>&1 - filesys_cmd_vrfy "ln" "$@" + filesys_cmd_vrfy "$LN_UTIL" "$@" { restore_shell_opts; } > /dev/null 2>&1 } diff --git a/ush/bash_utils/get_charvar_from_netcdf.sh b/ush/bash_utils/get_charvar_from_netcdf.sh index 71bcf9dd07..ac70cf68da 100644 --- a/ush/bash_utils/get_charvar_from_netcdf.sh +++ b/ush/bash_utils/get_charvar_from_netcdf.sh @@ -35,7 +35,7 @@ function get_charvar_from_netcdf() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -173,7 +173,7 @@ The arguments to this function are defined as follows: #----------------------------------------------------------------------- # nc_var_value=$( ncdump -v "${nc_var_name}" "${nc_file}" | \ - sed -r -e '1,/data:/d' \ + $SED -r -e '1,/data:/d' \ -e '/^[ ]*'${nc_var_name}'/d' \ -e '/^}$/d' \ -e 's/.*"(.*)".*/\1/' \ diff --git a/ush/bash_utils/get_elem_inds.sh b/ush/bash_utils/get_elem_inds.sh index 70da5b248e..89a9c41227 100644 --- a/ush/bash_utils/get_elem_inds.sh +++ b/ush/bash_utils/get_elem_inds.sh @@ -24,7 +24,7 @@ function get_elem_inds() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -123,7 +123,7 @@ The arguments to this function are defined as follows: # #----------------------------------------------------------------------- # - inds_to_return="${inds_to_return,,}" + inds_to_return=$(echo_lowercase $inds_to_return) valid_vals_inds_to_return=( "first" "last" "all" ) check_var_valid_value "inds_to_return" "valid_vals_inds_to_return" # diff --git a/ush/bash_utils/get_manage_externals_config_property.sh b/ush/bash_utils/get_manage_externals_config_property.sh index b42605ec9e..370d448f2a 100644 --- a/ush/bash_utils/get_manage_externals_config_property.sh +++ b/ush/bash_utils/get_manage_externals_config_property.sh @@ -24,7 +24,7 @@ function get_manage_externals_config_property() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -184,7 +184,7 @@ does not exist: #----------------------------------------------------------------------- # regex_search="^[ ]*(${property_name})[ ]*=[ ]*([^ ]*).*" - line=$( sed -r -n \ + line=$( $SED -r -n \ -e "/^[ ]*\[${external_name}\]/!b" \ -e ":SearchForLine" \ -e "s/(${regex_search})/\1/;t FoundLine" \ @@ -225,7 +225,7 @@ fied external (external_name): else property_value=$( printf "%s" "${line}" | \ - sed -r -n -e "s/${regex_search}/\2/p" ) + $SED -r -n -e "s/${regex_search}/\2/p" ) printf "%s\n" "${property_value}" fi diff --git a/ush/bash_utils/interpol_to_arbit_CRES.sh b/ush/bash_utils/interpol_to_arbit_CRES.sh index 6f685c1a54..bc4c6e8336 100644 --- a/ush/bash_utils/interpol_to_arbit_CRES.sh +++ b/ush/bash_utils/interpol_to_arbit_CRES.sh @@ -27,7 +27,7 @@ function interpol_to_arbit_CRES() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/is_array.sh b/ush/bash_utils/is_array.sh index 16be2fbb7a..c831f313e2 100644 --- a/ush/bash_utils/is_array.sh +++ b/ush/bash_utils/is_array.sh @@ -31,7 +31,7 @@ function is_array() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh index d3060db7d3..3ae9ce1cc5 100644 --- a/ush/bash_utils/is_element_of.sh +++ b/ush/bash_utils/is_element_of.sh @@ -24,7 +24,7 @@ function is_element_of() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 10bbe4ca63..72b7744b38 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -31,7 +31,7 @@ function print_input_args() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -62,7 +62,7 @@ function print_input_args() { # #----------------------------------------------------------------------- # - local caller_fp=$( readlink -f "${BASH_SOURCE[1]}" ) + local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) local caller_fn=$( basename "${caller_fp}" ) local caller_dir=$( dirname "${caller_fp}" ) local caller_name="${FUNCNAME[1]}" @@ -174,7 +174,7 @@ have been set as follows: # #----------------------------------------------------------------------- # - if [ ! -v VERBOSE ]; then + if [ -z ${VERBOSE+x} ]; then print_info_msg "$msg" else print_info_msg "$VERBOSE" "$msg" diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index 544357f628..d837329421 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -33,7 +33,7 @@ function print_info_msg() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -64,7 +64,7 @@ function print_info_msg() { # #----------------------------------------------------------------------- # - local caller_fp=$( readlink -f "${BASH_SOURCE[1]}" ) + local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) local caller_fn=$( basename "${caller_fp}" ) local caller_dir=$( dirname "${caller_fp}" ) local caller_name="${FUNCNAME[1]}" @@ -182,7 +182,7 @@ function print_err_msg_exit() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -213,7 +213,7 @@ function print_err_msg_exit() { # #----------------------------------------------------------------------- # - local caller_fp=$( readlink -f "${BASH_SOURCE[1]}" ) + local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) local caller_fn=$( basename "${caller_fp}" ) local caller_dir=$( dirname "${caller_fp}" ) local caller_name="${FUNCNAME[1]}" diff --git a/ush/bash_utils/process_args.sh b/ush/bash_utils/process_args.sh index 4363348bad..203bdedcf7 100644 --- a/ush/bash_utils/process_args.sh +++ b/ush/bash_utils/process_args.sh @@ -99,7 +99,7 @@ function process_args() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -263,7 +263,7 @@ ments (num_valid_args) specified in the array valid_arg_names: # Remove spaces (if any exist) from the current valid argument name. valid_arg_name_no_spaces=$( \ - printf "%s\n" "${valid_arg_name}" | sed -r -e 's/[[:space:]]//g' ) + printf "%s\n" "${valid_arg_name}" | $SED -r -e 's/[[:space:]]//g' ) if [ "${valid_arg_name_no_spaces}" != "${valid_arg_name}" ]; then print_err_msg_exit "\ diff --git a/ush/bash_utils/save_restore_shell_opts.sh b/ush/bash_utils/save_restore_shell_opts.sh index 37f699590f..b561b6cea3 100644 --- a/ush/bash_utils/save_restore_shell_opts.sh +++ b/ush/bash_utils/save_restore_shell_opts.sh @@ -19,7 +19,7 @@ function save_shell_opts() { # local shell_opts="$(set +o)"$'\n'"set -$-" shell_opts=${shell_opts//$'\n'/ } - shell_opts=$( printf "%s\n" "$shell_opts" | sed -r -e "s/set ([+-])/\1/g" ) + shell_opts=$( printf "%s\n" "$shell_opts" | $SED -r -e "s/set ([+-])/\1/g" ) # # Store the current set of shell options in the global array shell_- # opts_array so we can reuse them later. diff --git a/ush/bash_utils/set_bash_param.sh b/ush/bash_utils/set_bash_param.sh index 12c2e49be4..dd22206077 100644 --- a/ush/bash_utils/set_bash_param.sh +++ b/ush/bash_utils/set_bash_param.sh @@ -25,7 +25,7 @@ function set_bash_param() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -129,7 +129,7 @@ lar expression (regex_search): regex_search = ${regex_search}" }; - sed -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" + $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/set_file_param.sh b/ush/bash_utils/set_file_param.sh index 72a9ab041d..d454514831 100644 --- a/ush/bash_utils/set_file_param.sh +++ b/ush/bash_utils/set_file_param.sh @@ -25,7 +25,7 @@ function set_file_param() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -171,7 +171,7 @@ specified for this file: grep -q -E "${regex_search}" "${file_full_path}" if [ $? -eq 0 ]; then - sed -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" + $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" else print_err_msg_exit "\ Specified file (file_full_path) does not contain the searched-for regu- diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh index fe22d08abc..8d51cc01ba 100644 --- a/ush/check_ruc_lsm.sh +++ b/ush/check_ruc_lsm.sh @@ -30,7 +30,7 @@ function check_ruc_lsm() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -86,7 +86,7 @@ function check_ruc_lsm() { # ruc_lsm_name="lsm_ruc" regex_search="^[ ]*(${ruc_lsm_name})<\/scheme>[ ]*$" - ruc_lsm_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) + ruc_lsm_name_or_null=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) if [ "${ruc_lsm_name_or_null}" = "${ruc_lsm_name}" ]; then sdf_uses_ruc_lsm="TRUE" diff --git a/ush/cmp_expt_to_baseline.sh b/ush/cmp_expt_to_baseline.sh index 94e6e0c477..b7459593ab 100755 --- a/ush/cmp_expt_to_baseline.sh +++ b/ush/cmp_expt_to_baseline.sh @@ -28,7 +28,7 @@ module load nccmp # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/compare_config_scripts.sh b/ush/compare_config_scripts.sh index 279903ab35..791fa4e340 100644 --- a/ush/compare_config_scripts.sh +++ b/ush/compare_config_scripts.sh @@ -19,7 +19,7 @@ function compare_config_scripts() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -65,7 +65,7 @@ local func_name="${FUNCNAME[0]}" #----------------------------------------------------------------------- # var_list_default=$( \ -sed -r \ +$SED -r \ -e "s/^([ ]*)([^ ]+.*)/\2/g" \ -e "/^#.*/d" \ -e "/^$/d" \ @@ -73,7 +73,7 @@ sed -r \ ) var_list_local=$( \ -sed -r \ +$SED -r \ -e "s/^([ ]*)([^ ]+.*)/\2/g" \ -e "/^#.*/d" \ -e "/^$/d" \ @@ -95,7 +95,7 @@ while read crnt_line; do # Note that a variable name will be found only if the equal sign immed- # iately follows the variable name. # - var_name=$( printf "%s" "${crnt_line}" | sed -n -r -e "s/^([^ =\"]*)=.*/\1/p") + var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") if [ -z "${var_name}" ]; then diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 72c758df4a..dd6a2968c7 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -46,6 +46,11 @@ RUN_ENVIR="nco" # ACCOUNT: # The account under which to submit jobs to the queue. # +# WORKFLOW_MANAGER: +# The workflow manager to use (e.g. rocoto). This is set to "none" by +# default, but if the machine name is set to a platform that supports +# rocoto, this will be overwritten and set to "rocoto". +# # SCHED: # The job scheduler to use (e.g. slurm). Set this to an empty string in # order for the experiment generation script to set it depending on the @@ -103,6 +108,7 @@ RUN_ENVIR="nco" # MACHINE="BIG_COMPUTER" ACCOUNT="project_name" +WORKFLOW_MANAGER="none" SCHED="" PARTITION_DEFAULT="" QUEUE_DEFAULT="" @@ -113,6 +119,30 @@ QUEUE_FCST="" # #----------------------------------------------------------------------- # +# Set run commands for platforms without a workflow manager. These values +# will be ignored unless WORKFLOW_MANAGER="none". Definitions: +# +# RUN_CMD_UTILS: +# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, etc.) +# Can be left blank for smaller domains, in which case the executables will run +# without MPI. +# +# RUN_CMD_FCST: +# The run command for the model forecast step. This will be appended to the end +# of the variable definitions file, so it can reference other variables. +# +# RUN_CMD_POST: +# The run command for post-processing (UPP). Can be left blank for smaller +# domains, in which case UPP will run without MPI. +# +#----------------------------------------------------------------------- +# +RUN_CMD_UTILS="mpirun -np 1" +RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}" +RUN_CMD_POST="mpirun -np 1" +# +#----------------------------------------------------------------------- +# # Set cron-associated parameters. Definitions: # # USE_CRON_TO_RELAUNCH: diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 10ad396ce9..3c409031a6 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -25,7 +25,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index 23602932bb..ccd55d00fa 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -25,7 +25,7 @@ function create_model_configure_file() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -99,10 +99,10 @@ run directory (run_dir): # # Set parameters in the model configure file. # - dot_quilting_dot="."${QUILTING,,}"." - dot_print_esmf_dot="."${PRINT_ESMF,,}"." - dot_cpl_dot="."${CPL,,}"." - dot_write_dopost="."${WRITE_DOPOST,,}"." + dot_quilting_dot="."$(echo_lowercase $QUILTING)"." + dot_print_esmf_dot="."$(echo_lowercase $PRINT_ESMF)"." + dot_cpl_dot="."$(echo_lowercase $CPL)"." + dot_write_dopost="."$(echo_lowercase $WRITE_DOPOST)"." # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index e5d2f496ed..0ee8eee0b8 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -19,7 +19,11 @@ function generate_FV3LAM_wflow() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +if [[ $(uname -s) == Darwin ]]; then + local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) + else + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + fi local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -408,7 +412,7 @@ settings="\ # 'sub_hourly_post': ${SUB_HOURLY_POST} 'delta_min': ${DT_SUBHOURLY_POST_MNTS} - 'first_fv3_file_tstr': "000:"`date -d "${DATE_FIRST_CYCL} +${DT_ATMOS} seconds" +%M:%S` + 'first_fv3_file_tstr': "000:"`$DATE_UTIL -d "${DATE_FIRST_CYCL} +${DT_ATMOS} seconds" +%M:%S` " # End of "settings" variable. print_info_msg $VERBOSE " @@ -423,12 +427,13 @@ $settings" # script to generate the experiment's actual XML file from this template # file. # -template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" -$USHDIR/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${template_xml_fp} \ - -o ${WFLOW_XML_FP} || \ - print_err_msg_exit "\ +if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then + template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" + $USHDIR/fill_jinja_template.py -q \ + -u "${settings}" \ + -t ${template_xml_fp} \ + -o ${WFLOW_XML_FP} || \ + print_err_msg_exit "\ Call to python script fill_jinja_template.py to create a rocoto workflow XML file from a template file failed. Parameters passed to this script are: @@ -439,6 +444,7 @@ are: Namelist settings specified on command line: settings = $settings" +fi # #----------------------------------------------------------------------- # @@ -468,7 +474,7 @@ if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then # # Make a backup copy of the user's crontab file and save it in a file. # - time_stamp=$( date "+%F_%T" ) + time_stamp=$( $DATE_UTIL "+%F_%T" ) crontab_backup_fp="$EXPTDIR/crontab.bak.${time_stamp}" print_info_msg " Copying contents of user cron table to backup file: @@ -485,7 +491,7 @@ Copying contents of user cron table to backup file: # CRONTAB_LINE with backslashes. Do this next. # crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ - sed -r -e "s%[*]%\\\\*%g" ) + $SED -r -e "s%[*]%\\\\*%g" ) # # In the grep command below, the "^" at the beginning of the string be- # ing passed to grep is a start-of-line anchor while the "$" at the end @@ -549,7 +555,7 @@ if [ "${RUN_ENVIR}" = "nco" ]; then # Resolve the target directory that the FIXam symlink points to and check # that it exists. # - path_resolved=$( readlink -m "$FIXam" ) + path_resolved=$( $READLINK -m "$FIXam" ) if [ ! -d "${path_resolved}" ]; then print_err_msg_exit "\ In order to be able to generate a forecast experiment in NCO mode (i.e. @@ -762,9 +768,9 @@ for (( i=0; i<${num_nml_vars}; i++ )); do mapping="${FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[$i]}" nml_var_name=$( printf "%s\n" "$mapping" | \ - sed -n -r -e "s/${regex_search}/\1/p" ) + $SED -n -r -e "s/${regex_search}/\1/p" ) FIXam_fn=$( printf "%s\n" "$mapping" | - sed -n -r -e "s/${regex_search}/\2/p" ) + $SED -n -r -e "s/${regex_search}/\2/p" ) fp="\"\"" if [ ! -z "${FIXam_fn}" ]; then @@ -870,9 +876,11 @@ cp_vrfy $USHDIR/${EXPT_CONFIG_FN} $EXPTDIR # #----------------------------------------------------------------------- # -wflow_db_fn="${WFLOW_XML_FN%.xml}.db" -rocotorun_cmd="rocotorun -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" -rocotostat_cmd="rocotostat -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" +if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then + wflow_db_fn="${WFLOW_XML_FN%.xml}.db" + rocotorun_cmd="rocotorun -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" + rocotostat_cmd="rocotostat -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" +fi print_info_msg " ======================================================================== @@ -888,25 +896,20 @@ The experiment directory is: > EXPTDIR=\"$EXPTDIR\" " -case "$MACHINE" in - -"CHEYENNE") +# +#----------------------------------------------------------------------- +# +# If rocoto is required, print instructions on how to load and use it +# +#----------------------------------------------------------------------- +# +if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then print_info_msg "\ To launch the workflow, first ensure that you have a compatible version -of rocoto in your \$PATH. On Cheyenne, version 1.3.1 has been pre-built; -you can load it in your \$PATH with one of the following commands, depending -on your default shell: - -bash: - > export PATH=\${PATH}:/glade/p/ral/jntp/tools/rocoto/rocoto-1.3.1/bin/ - -tcsh: - > setenv PATH \${PATH}:/glade/p/ral/jntp/tools/rocoto/rocoto-1.3.1/bin/ -" - ;; - -*) - print_info_msg "\ +of rocoto available. For most pre-configured platforms, rocoto can be +loaded via a module: + > module load rocoto +For more details on rocoto, see the User's Guide. To launch the workflow, first ensure that you have a compatible version of rocoto loaded. For example, to load version 1.3.1 of rocoto, use @@ -914,11 +917,7 @@ of rocoto loaded. For example, to load version 1.3.1 of rocoto, use (This version has been tested on hera; later versions may also work but have not been tested.) -" - ;; -esac -print_info_msg " To launch the workflow, change location to the experiment directory (EXPTDIR) and issue the rocotrun command, as follows: @@ -939,7 +938,9 @@ Note that: 2) In order for the output of the rocotostat command to be up-to-date, the rocotorun command must be issued immediately before the rocoto- stat command. - +" +fi +print_info_msg " For automatic resubmission of the workflow (say every 3 minutes), the following line can be added to the user's crontab (use \"crontab -e\" to edit the cron table): @@ -991,7 +992,12 @@ set -u # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +if [[ $(uname -s) == Darwin ]]; then + command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) +else + scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +fi scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -1052,7 +1058,7 @@ rm "${tmp_fp}" # ful, move the log file in which the "tee" command saved the output of # the function to the experiment directory. # -if [ $retval -eq 0 ]; then +if [[ $retval == 0 ]]; then mv "${log_fp}" "$exptdir" # # If the call to the generate_FV3LAM_wflow function above was not suc- diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index f2224559ae..ff7d1bfda1 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -37,7 +37,7 @@ function get_extrn_mdl_file_dir_info() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -209,11 +209,27 @@ fi # #----------------------------------------------------------------------- # +# Declare local function to avoid repetition +# +#----------------------------------------------------------------------- +# +function quit_unless_user_spec_data() { + if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then + print_err_msg_exit "\ +The system directory in which to look for external model output files +has not been specified for this external model and machine combination: + extrn_mdl_name = \"${extrn_mdl_name}\" + MACHINE = \"$MACHINE\"" + fi +} +# +#----------------------------------------------------------------------- +# # Check input variables for valid values. # #----------------------------------------------------------------------- # - anl_or_fcst="${anl_or_fcst^^}" + anl_or_fcst=$(echo_uppercase $anl_or_fcst) valid_vals_anl_or_fcst=( "ANL" "FCST" ) check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" # @@ -234,7 +250,7 @@ fi hh=${cdate_FV3LAM:8:2} yyyymmdd=${cdate_FV3LAM:0:8} - cdate=$( date --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) + cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) # #----------------------------------------------------------------------- # @@ -298,7 +314,7 @@ fi # Get the Julian day-of-year of the starting date and time of the exter- # nal model forecast. # - ddd=$( date --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) + ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) # # Get the last two digits of the year of the starting date and time of # the external model forecast. @@ -434,12 +450,14 @@ fi ;; *) - print_err_msg_exit "\ + if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then + print_err_msg_exit "\ The external model file names (either on disk or in archive files) have not yet been specified for this combination of external model (extrn_mdl_name) and analysis or forecast (anl_or_fcst): extrn_mdl_name = \"${extrn_mdl_name}\" anl_or_fcst = \"${anl_or_fcst}\"" + fi ;; esac @@ -566,12 +584,14 @@ and analysis or forecast (anl_or_fcst): ;; *) - print_err_msg_exit "\ + if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then + print_err_msg_exit "\ The external model file names have not yet been specified for this com- bination of external model (extrn_mdl_name) and analysis or forecast (anl_or_fcst): extrn_mdl_name = \"${extrn_mdl_name}\" anl_or_fcst = \"${anl_or_fcst}\"" + fi ;; esac @@ -595,6 +615,7 @@ bination of external model (extrn_mdl_name) and analysis or forecast sysbasedir="${EXTRN_MDL_SYSBASEDIR_LBCS}" fi + sysdir="" case "${extrn_mdl_name}" in # @@ -628,11 +649,7 @@ bination of external model (extrn_mdl_name) and analysis or forecast sysdir="$sysbasedir" ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" + quit_unless_user_spec_data ;; esac ;; @@ -665,11 +682,7 @@ has not been specified for this external model and machine combination: sysdir="$sysbasedir" ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" + quit_unless_user_spec_data ;; esac ;; @@ -699,11 +712,7 @@ has not been specified for this external model and machine combination: sysdir="$sysbasedir" ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" + quit_unless_user_spec_data ;; esac ;; @@ -733,11 +742,7 @@ has not been specified for this external model and machine combination: sysdir="$sysbasedir" ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" + quit_unless_user_spec_data ;; esac ;; @@ -766,22 +771,14 @@ has not been specified for this external model and machine combination: sysdir="$sysbasedir" ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" + quit_unless_user_spec_data ;; esac ;; *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - + quit_unless_user_spec_data esac # #----------------------------------------------------------------------- diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 453db87d3b..72cdd74c77 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -17,7 +17,12 @@ set -u # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +if [[ $(uname -s) == Darwin ]]; then + command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) +else + scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +fi scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -65,7 +70,12 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # exptdir=$( dirname "$0" ) -exptdir=$( readlink -f "$exptdir" ) +if [[ $(uname -s) == Darwin ]]; then + command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + exptdir=$( greadlink -f "$exptdir" ) +else + exptdir=$( readlink -f "$exptdir" ) +fi # #----------------------------------------------------------------------- # @@ -395,7 +405,7 @@ launch script for this experiment: # CRONTAB_LINE with backslashes. Do this next. # crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ - sed -r -e "s%[*]%\\\\*%g" ) + $SED -r -e "s%[*]%\\\\*%g" ) # # In the string passed to the grep command below, we use the line start # and line end anchors ("^" and "$", respectively) to ensure that we on- diff --git a/ush/link_fix.sh b/ush/link_fix.sh index e9a641d962..34bcd5a53a 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -26,7 +26,7 @@ function link_fix() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -289,7 +289,7 @@ Creating links in the FIXLAM directory to the grid files..." fn=$( basename $fp ) - res=$( printf "%s" $fn | sed -n -r -e "s/^C([0-9]*).*/\1/p" ) + res=$( printf "%s" $fn | $SED -n -r -e "s/^C([0-9]*).*/\1/p" ) if [ -z $res ]; then print_err_msg_exit "\ The resolution could not be extracted from the current file's name. The diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 8b91ef0cfe..411f0bdb1e 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -152,7 +152,7 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # -machine=${MACHINE,,} +machine=$(echo_lowercase $MACHINE) env_fn="build_${machine}_${COMPILER}.env" env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" source "${env_fp}" || print_err_msg_exit "\ @@ -191,89 +191,6 @@ modulefile_name="${task_name}" default_modules_dir="$HOMErrfs/modulefiles" default_modulefile_name="${machine}.default" use_default_modulefile=0 -####### -####### The following lines (199-276) can be removed once we confirm -####### that the new method of setting environment variables and loading -####### modules will remain permanent. -####### -# -#----------------------------------------------------------------------- -# -# This comment needs to be updated: -# -# Use the "readlink" command to resolve the full path to the module file -# and then verify that the file exists. This is not necessary for most -# tasks, but for the run_fcst task, when CCPP is enabled, the module -# file in the modules directory is not a regular file but a symlink to a -# file in the ufs_weather_model external repo. This latter target file -# will exist only if the forecast model code has already been built. -# Thus, we now check to make sure that the module file exits. -# -#----------------------------------------------------------------------- -# -#if [ "${machine}" = "unknown" ]; then -# -# This if-statement allows for a graceful exit in the case in which module -# files are not needed for the task. -# This is not currently used but reserved for future development. -# -# print_info_msg " -#Module files are not needed for this task (task_name) and machine (machine): -# task_name = \"${task_name}\" -# machine = \"${machine}\"" - -#else - -# modulefile_path=$( readlink -f "${modules_dir}/${modulefile_name}" ) - -# if [ ! -f "${modulefile_path}" ]; then - -# default_modulefile_path=$( readlink -f "${default_modules_dir}/${default_modulefile_name}" ) -# if [ -f "${default_modulefile_path}" ]; then -# -# If the task-specific modulefile does not exist but a default one does, -# use it! -# -# print_info_msg "$VERBOSE" " -#A task-specific modulefile (modulefile_path) does not exist for this task -#(task_name) and machine (machine) combination: -# task_name = \"${task_name}\" -# machine = \"${machine}\" -# modulefile_path = \"${modulefile_path}\" -#Will attempt to use the default modulefile (default_modulefile_path): -# default_modulefile_path = \"${default_modulefile_path}\"" -# -# modules_dir="${default_modules_dir}" -# use_default_modulefile=1 -# -# elif [ "${task_name}" = "${MAKE_OROG_TN}" ] || \ -# [ "${task_name}" = "${MAKE_SFC_CLIMO_TN}" ] || \ -# [ "${task_name}" = "${MAKE_ICS_TN}" ] || \ -# [ "${task_name}" = "${MAKE_LBCS_TN}" ] || \ -# [ "${task_name}" = "${RUN_FCST_TN}" ]; then -# -# print_err_msg_exit "\ -#The target (modulefile_path) of the symlink (modulefile_name) in the task -#modules directory (modules_dir) that points to module file for this task -#(task_name) does not exist: -# task_name = \"${task_name}\" -# modulefile_name = \"${modulefile_name}\" -# modules_dir = \"${modules_dir}\" -# modulefile_path = \"${modulefile_path}\" -#This is likely because the forecast model code has not yet been built." -# -# else -# -# print_err_msg_exit "\ -#The module file (modulefile_path) specified for this task (task_name) -#does not exist: -# task_name = \"${task_name}\" -# modulefile_path = \"${modulefile_path}\" -# machine = \"${machine}\"" -# -# fi -# -# fi # #----------------------------------------------------------------------- # diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 0026cb86af..8a81feb146 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -27,7 +27,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh index 2d44a9d0c2..13f2ecdf5d 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ b/ush/set_FV3nml_sfc_climo_filenames.sh @@ -30,7 +30,7 @@ function set_FV3nml_sfc_climo_filenames() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -145,9 +145,9 @@ for (( i=0; i<${num_nml_vars}; i++ )); do mapping="${FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING[$i]}" nml_var_name=$( printf "%s\n" "$mapping" | \ - sed -n -r -e "s/${regex_search}/\1/p" ) + $SED -n -r -e "s/${regex_search}/\1/p" ) sfc_climo_field_name=$( printf "%s\n" "$mapping" | - sed -n -r -e "s/${regex_search}/\2/p" ) + $SED -n -r -e "s/${regex_search}/\2/p" ) # # Check that the surface climatology field associated with the current # namelist variable is valid. diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index a6d4c6dfd1..cfe115d525 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -32,7 +32,7 @@ function set_FV3nml_stoch_params() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/set_cycle_dates.sh b/ush/set_cycle_dates.sh index b8d406c128..c4b3adfbf1 100644 --- a/ush/set_cycle_dates.sh +++ b/ush/set_cycle_dates.sh @@ -30,7 +30,7 @@ function set_cycle_dates() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -106,7 +106,7 @@ End date (date_end) must be at or after start date (date_start): date_crnt="${date_start}" while [ "${date_crnt}" -le "${date_end}" ]; do all_cdates+=( $( printf "%s " ${cycle_hrs[@]/#/${date_crnt}} ) ) - date_crnt=$( date -d "${date_crnt} + 1 days" +%Y%m%d ) + date_crnt=$( $DATE_UTIL -d "${date_crnt} + 1 days" +%Y%m%d ) done # #----------------------------------------------------------------------- diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 72855d9fb3..c40e3a1280 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -17,7 +17,7 @@ function set_extrn_mdl_params() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/set_gridparams_ESGgrid.sh b/ush/set_gridparams_ESGgrid.sh index 91ec8b83df..41efbfe582 100644 --- a/ush/set_gridparams_ESGgrid.sh +++ b/ush/set_gridparams_ESGgrid.sh @@ -26,7 +26,7 @@ function set_gridparams_ESGgrid() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/set_gridparams_GFDLgrid.sh b/ush/set_gridparams_GFDLgrid.sh index 7d47affb1c..a76b9a58a0 100644 --- a/ush/set_gridparams_GFDLgrid.sh +++ b/ush/set_gridparams_GFDLgrid.sh @@ -26,7 +26,7 @@ function set_gridparams_GFDLgrid() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -425,8 +425,8 @@ AFTER adjustments are: nx_of_t6_on_t6sg=$(( 2*nx_of_t6_on_t6g )) ny_of_t6_on_t6sg=$(( 2*ny_of_t6_on_t6g )) - prime_factors_nx_of_t7_on_t7g=$( factor ${nx_of_t7_on_t7g} | sed -r -e 's/^[0-9]+: (.*)/\1/' ) - prime_factors_ny_of_t7_on_t7g=$( factor ${ny_of_t7_on_t7g} | sed -r -e 's/^[0-9]+: (.*)/\1/' ) + prime_factors_nx_of_t7_on_t7g=$( factor ${nx_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) + prime_factors_ny_of_t7_on_t7g=$( factor ${ny_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) print_info_msg "$VERBOSE" " The number of cells in the two horizontal directions (x and y) on the diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index 5fb90f8394..c857492798 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -44,7 +44,7 @@ function set_ozone_param() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -130,7 +130,7 @@ function set_ozone_param() { #----------------------------------------------------------------------- # regex_search="^[ ]*(ozphys.*)<\/scheme>[ ]*$" - ozone_param=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) + ozone_param=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) if [ "${ozone_param}" = "ozphys_2015" ]; then fixgsm_ozone_fn="ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77" @@ -175,11 +175,11 @@ num_symlinks=${#CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]} for (( i=0; i<${num_symlinks}; i++ )); do mapping="${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]}" symlink=$( printf "%s\n" "$mapping" | \ - sed -n -r -e "s/${regex_search}/\1/p" ) + $SED -n -r -e "s/${regex_search}/\1/p" ) if [ "$symlink" = "${ozone_symlink}" ]; then regex_search="^[ ]*([^| ]+[ ]*)[|][ ]*([^| ]*)[ ]*$" mapping_ozone=$( printf "%s\n" "$mapping" | \ - sed -n -r -e "s/${regex_search}/\1/p" ) + $SED -n -r -e "s/${regex_search}/\1/p" ) mapping_ozone="${mapping_ozone}| ${fixgsm_ozone_fn}" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]="${mapping_ozone}" fixgsm_ozone_fn_is_set="TRUE" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 698a7e2a8a..aca2a26e36 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -25,7 +25,7 @@ function set_predef_grid_params() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 558333435b..667ad522fa 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -32,7 +32,7 @@ function set_thompson_mp_fix_files() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -93,7 +93,7 @@ function set_thompson_mp_fix_files() { # thompson_mp_name="mp_thompson" regex_search="^[ ]*(${thompson_mp_name})<\/scheme>[ ]*$" - thompson_mp_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) + thompson_mp_name_or_null=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) if [ "${thompson_mp_name_or_null}" = "${thompson_mp_name}" ]; then sdf_uses_thompson_mp="TRUE" diff --git a/ush/setup.sh b/ush/setup.sh index bc11c67b40..96b1ef0717 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -23,7 +23,7 @@ function setup() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -140,7 +140,7 @@ check_var_valid_value "VERBOSE" "valid_vals_VERBOSE" # Set VERBOSE to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -VERBOSE=${VERBOSE^^} +VERBOSE=$(echo_uppercase $VERBOSE) if [ "$VERBOSE" = "TRUE" ] || \ [ "$VERBOSE" = "YES" ]; then VERBOSE="TRUE" @@ -160,7 +160,7 @@ check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_USE_CRON_TO_RELAUNCH" # Set USE_CRON_TO_RELAUNCH to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -USE_CRON_TO_RELAUNCH=${USE_CRON_TO_RELAUNCH^^} +USE_CRON_TO_RELAUNCH=$(echo_uppercase $USE_CRON_TO_RELAUNCH) if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ] || \ [ "${USE_CRON_TO_RELAUNCH}" = "YES" ]; then USE_CRON_TO_RELAUNCH="TRUE" @@ -180,7 +180,7 @@ check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_RUN_TASK_MAKE_GRID" # Set RUN_TASK_MAKE_GRID to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_MAKE_GRID=${RUN_TASK_MAKE_GRID^^} +RUN_TASK_MAKE_GRID=$(echo_uppercase $RUN_TASK_MAKE_GRID) if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_GRID}" = "YES" ]; then RUN_TASK_MAKE_GRID="TRUE" @@ -200,7 +200,7 @@ check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_RUN_TASK_MAKE_OROG" # Set RUN_TASK_MAKE_OROG to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_MAKE_OROG=${RUN_TASK_MAKE_OROG^^} +RUN_TASK_MAKE_OROG=$(echo_uppercase $RUN_TASK_MAKE_OROG) if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_OROG}" = "YES" ]; then RUN_TASK_MAKE_OROG="TRUE" @@ -221,7 +221,7 @@ check_var_valid_value \ # Set RUN_TASK_MAKE_SFC_CLIMO to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -RUN_TASK_MAKE_SFC_CLIMO=${RUN_TASK_MAKE_SFC_CLIMO^^} +RUN_TASK_MAKE_SFC_CLIMO=$(echo_uppercase $RUN_TASK_MAKE_SFC_CLIMO) if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] || \ [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "YES" ]; then RUN_TASK_MAKE_SFC_CLIMO="TRUE" @@ -242,7 +242,7 @@ check_var_valid_value \ # Set RUN_TASK_RUN_POST to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -RUN_TASK_RUN_POST=${RUN_TASK_RUN_POST^^} +RUN_TASK_RUN_POST=$(echo_uppercase $RUN_TASK_RUN_POST) if [ "${RUN_TASK_RUN_POST}" = "TRUE" ] || \ [ "${RUN_TASK_RUN_POST}" = "YES" ]; then RUN_TASK_RUN_POST="TRUE" @@ -262,7 +262,7 @@ check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_RUN_TASK_VX_GRIDSTAT" # Set RUN_TASK_VX_GRIDSTAT to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_VX_GRIDSTAT=${RUN_TASK_VX_GRIDSTAT^^} +RUN_TASK_VX_GRIDSTAT=$(echo_uppercase $RUN_TASK_VX_GRIDSTAT) if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ [ "${RUN_TASK_VX_GRIDSTAT}" = "YES" ]; then RUN_TASK_VX_GRIDSTAT="TRUE" @@ -282,7 +282,7 @@ check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_RUN_TASK_VX_POINTSTAT" # Set RUN_TASK_VX_POINTSTAT to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_VX_POINTSTAT=${RUN_TASK_VX_POINTSTAT^^} +RUN_TASK_VX_POINTSTAT=$(echo_uppercase $RUN_TASK_VX_POINTSTAT) if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ [ "${RUN_TASK_VX_POINTSTAT}" = "YES" ]; then RUN_TASK_VX_POINTSTAT="TRUE" @@ -303,7 +303,7 @@ check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_RUN_TASK_VX_ENSGRID" # Set RUN_TASK_VX_ENSGRID to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_VX_ENSGRID=${RUN_TASK_VX_ENSGRID^^} +RUN_TASK_VX_ENSGRID=$(echo_uppercase $RUN_TASK_VX_ENSGRID) if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ [ "${RUN_TASK_VX_ENSGRID}" = "YES" ]; then RUN_TASK_VX_ENSGRID="TRUE" @@ -325,7 +325,7 @@ check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_RUN_TASK_VX_ENSPOINT" # Set RUN_TASK_VX_ENSPOINT to either "TRUE" or "FALSE" so we don't have to # consider other valid values later on. # -RUN_TASK_VX_ENSPOINT=${RUN_TASK_VX_ENSPOINT^^} +RUN_TASK_VX_ENSPOINT=$(echo_uppercase $RUN_TASK_VX_ENSPOINT) if [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ] || \ [ "${RUN_TASK_VX_ENSPOINT}" = "YES" ]; then RUN_TASK_VX_ENSPOINT="TRUE" @@ -345,7 +345,7 @@ check_var_valid_value "DO_SHUM" "valid_vals_DO_SHUM" # Set DO_SHUM to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -DO_SHUM=${DO_SHUM^^} +DO_SHUM=$(echo_uppercase $DO_SHUM) if [ "${DO_SHUM}" = "TRUE" ] || \ [ "${DO_SHUM}" = "YES" ]; then DO_SHUM="TRUE" @@ -365,7 +365,7 @@ check_var_valid_value "DO_SPPT" "valid_vals_DO_SPPT" # Set DO_SPPT to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -DO_SPPT=${DO_SPPT^^} +DO_SPPT=$(echo_uppercase $DO_SPPT) if [ "${DO_SPPT}" = "TRUE" ] || \ [ "${DO_SPPT}" = "YES" ]; then DO_SPPT="TRUE" @@ -385,7 +385,7 @@ check_var_valid_value "DO_SKEB" "valid_vals_DO_SKEB" # Set DO_SKEB to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -DO_SKEB=${DO_SKEB^^} +DO_SKEB=$(echo_uppercase $DO_SKEB) if [ "${DO_SKEB}" = "TRUE" ] || \ [ "${DO_SKEB}" = "YES" ]; then DO_SKEB="TRUE" @@ -405,7 +405,7 @@ check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" # Set DO_SPP to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -DO_SPP=${DO_SPP^^} +DO_SPP=$(echo_uppercase $DO_SPP) if [ "${DO_SPP}" = "TRUE" ] || \ [ "${DO_SPP}" = "YES" ]; then DO_SPP="TRUE" @@ -458,7 +458,7 @@ check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" # Set USE_FVCOM to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -USE_FVCOM=${USE_FVCOM^^} +USE_FVCOM=$(echo_uppercase $USE_FVCOM) if [ "$USE_FVCOM" = "TRUE" ] || \ [ "$USE_FVCOM" = "YES" ]; then USE_FVCOM="TRUE" @@ -479,7 +479,7 @@ check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" # Set SUB_HOURLY_POST to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -SUB_HOURLY_POST=${SUB_HOURLY_POST^^} +SUB_HOURLY_POST=$(echo_uppercase $SUB_HOURLY_POST) if [ "${SUB_HOURLY_POST}" = "TRUE" ] || \ [ "${SUB_HOURLY_POST}" = "YES" ]; then SUB_HOURLY_POST="TRUE" @@ -503,7 +503,7 @@ check_var_valid_value "DOT_OR_USCORE" "valid_vals_DOT_OR_USCORE" # #----------------------------------------------------------------------- # -MACHINE=$( printf "%s" "$MACHINE" | sed -e 's/\(.*\)/\U\1/' ) +MACHINE=$( printf "%s" "$MACHINE" | $SED -e 's/\(.*\)/\U\1/' ) check_var_valid_value "MACHINE" "valid_vals_MACHINE" # #----------------------------------------------------------------------- @@ -520,10 +520,11 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" #----------------------------------------------------------------------- # RELATIVE_LINK_FLAG="" - -case "$MACHINE" in +NCORES_PER_NODE="2" # Need some arbitrary default value to avoid division by zero errors +case $MACHINE in "WCOSS_CRAY") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE="24" SCHED="lsfcray" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} @@ -534,6 +535,7 @@ case "$MACHINE" in ;; "WCOSS_DELL_P3") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=24 SCHED="lsf" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} @@ -544,6 +546,7 @@ case "$MACHINE" in ;; "HERA") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=40 SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} @@ -557,6 +560,7 @@ case "$MACHINE" in ;; "ORION") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=40 SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} @@ -570,6 +574,7 @@ case "$MACHINE" in ;; "JET") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=24 SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} @@ -583,6 +588,7 @@ case "$MACHINE" in ;; "ODIN") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=24 SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} @@ -596,6 +602,7 @@ case "$MACHINE" in ;; "CHEYENNE") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=36 SCHED=${SCHED:-"pbspro"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} @@ -606,6 +613,7 @@ case "$MACHINE" in ;; "STAMPEDE") + WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=68 SCHED="slurm" PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} @@ -618,6 +626,16 @@ case "$MACHINE" in RELATIVE_LINK_FLAG="--relative" ;; + "MACOS") + WORKFLOW_MANAGER="none" + SCHED="none" + ;; + + "LINUX") + WORKFLOW_MANAGER="none" + SCHED="none" + ;; + esac # #----------------------------------------------------------------------- @@ -635,20 +653,27 @@ PPN_RUN_FCST=${PPN_RUN_FCST:-${PPN_RUN_FCST_OPT}} # #----------------------------------------------------------------------- # -SCHED="${SCHED,,}" +SCHED=$(echo_lowercase $SCHED) check_var_valid_value "SCHED" "valid_vals_SCHED" # #----------------------------------------------------------------------- # -# Verify that the ACCOUNT variable is not empty. If it is, print out an -# error message and exit. +# If we are using a workflow manager, run some checks. First, +# verify that the ACCOUNT variable is not empty. Second, ensure that the +# custom RUN_CMD variables are not set. # #----------------------------------------------------------------------- # -if [ -z "$ACCOUNT" ]; then - print_err_msg_exit "\ -The variable ACCOUNT cannot be empty: - ACCOUNT = \"$ACCOUNT\"" +if [ "$WORKFLOW_MANAGER" != "none" ]; then + if [ -z "$ACCOUNT" ]; then + print_err_msg_exit "\ +The variable ACCOUNT cannot be empty if you are using a workflow manager: + ACCOUNT = \"$ACCOUNT\" + WORKFLOW_MANAGER = \"$WORKFLOW_MANAGER\"" + fi + RUN_CMD_UTILS="" + RUN_CMD_FCST="" + RUN_CMD_POST="" fi # #----------------------------------------------------------------------- @@ -753,7 +778,7 @@ fi #----------------------------------------------------------------------- # DATE_OR_NULL=$( printf "%s" "${DATE_FIRST_CYCL}" | \ - sed -n -r -e "s/^([0-9]{8})$/\1/p" ) + $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) if [ -z "${DATE_OR_NULL}" ]; then print_err_msg_exit "\ DATE_FIRST_CYCL must be a string consisting of exactly 8 digits of the @@ -763,7 +788,7 @@ month, and DD is the 2-digit day-of-month. fi DATE_OR_NULL=$( printf "%s" "${DATE_LAST_CYCL}" | \ - sed -n -r -e "s/^([0-9]{8})$/\1/p" ) + $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) if [ -z "${DATE_OR_NULL}" ]; then print_err_msg_exit "\ DATE_LAST_CYCL must be a string consisting of exactly 8 digits of the @@ -785,7 +810,7 @@ CYCL_HRS_str="( $CYCL_HRS_str)" i=0 for CYCL in "${CYCL_HRS[@]}"; do - CYCL_OR_NULL=$( printf "%s" "$CYCL" | sed -n -r -e "s/^([0-9]{2})$/\1/p" ) + CYCL_OR_NULL=$( printf "%s" "$CYCL" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) if [ -z "${CYCL_OR_NULL}" ]; then print_err_msg_exit "\ @@ -955,7 +980,8 @@ case "$MACHINE" in ;; *) - print_err_msg_exit "\ + if [ -z "$FIXgsm" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then + print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} @@ -963,6 +989,7 @@ One or more fix file directories have not been specified for this machine: SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} You can specify the missing location(s) in config.sh" + fi ;; esac @@ -979,7 +1006,7 @@ esac # #----------------------------------------------------------------------- # -mng_extrns_cfg_fn=$( readlink -f "${SR_WX_APP_TOP_DIR}/Externals.cfg" ) +mng_extrns_cfg_fn=$( $READLINK -f "${SR_WX_APP_TOP_DIR}/Externals.cfg" ) property_name="local_path" # # Get the base directory of the FV3 forecast model code. @@ -1051,7 +1078,7 @@ check_var_valid_value \ # Set USE_CUSTOM_POST_CONFIG_FILE to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -USE_CUSTOM_POST_CONFIG_FILE=${USE_CUSTOM_POST_CONFIG_FILE^^} +USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase $USE_CUSTOM_POST_CONFIG_FILE) if [ "$USE_CUSTOM_POST_CONFIG_FILE" = "TRUE" ] || \ [ "$USE_CUSTOM_POST_CONFIG_FILE" = "YES" ]; then USE_CUSTOM_POST_CONFIG_FILE="TRUE" @@ -1224,7 +1251,7 @@ if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then # digits. # mnts_or_null=$( printf "%s" "${DT_SUBHOURLY_POST_MNTS}" | \ - sed -n -r -e "s/^([0-9])([0-9])?$/\1\2/p" ) + $SED -n -r -e "s/^([0-9])([0-9])?$/\1\2/p" ) if [ -z "${mnts_or_null}" ]; then print_err_msg_exit "\ When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), @@ -1318,7 +1345,7 @@ fi if [ "${EXPT_BASEDIR:0:1}" != "/" ]; then EXPT_BASEDIR="${SR_WX_APP_TOP_DIR}/../expt_dirs/${EXPT_BASEDIR}" fi -EXPT_BASEDIR="$( readlink -m ${EXPT_BASEDIR} )" +EXPT_BASEDIR="$( $READLINK -m ${EXPT_BASEDIR} )" mkdir_vrfy -p "${EXPT_BASEDIR}" # #----------------------------------------------------------------------- @@ -1577,7 +1604,7 @@ check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_USE_USER_STAGED_ # Set USE_USER_STAGED_EXTRN_FILES to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -USE_USER_STAGED_EXTRN_FILES=${USE_USER_STAGED_EXTRN_FILES^^} +USE_USER_STAGED_EXTRN_FILES=$(echo_uppercase $USE_USER_STAGED_EXTRN_FILES) if [ "${USE_USER_STAGED_EXTRN_FILES}" = "YES" ]; then USE_USER_STAGED_EXTRN_FILES="TRUE" elif [ "${USE_USER_STAGED_EXTRN_FILES}" = "NO" ]; then @@ -1624,7 +1651,7 @@ check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" # Set DO_ENSEMBLE to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -DO_ENSEMBLE=${DO_ENSEMBLE^^} +DO_ENSEMBLE=$(echo_uppercase $DO_ENSEMBLE) if [ "$DO_ENSEMBLE" = "TRUE" ] || \ [ "$DO_ENSEMBLE" = "YES" ]; then DO_ENSEMBLE="TRUE" @@ -2125,12 +2152,14 @@ fi #----------------------------------------------------------------------- # # Create a new experiment directory. Note that at this point we are -# guaranteed that there is no preexisting experiment directory. +# guaranteed that there is no preexisting experiment directory. For +# platforms with no workflow manager, we need to create LOGDIR as well, +# since it won't be created later at runtime. # #----------------------------------------------------------------------- # mkdir_vrfy -p "$EXPTDIR" - +mkdir_vrfy -p "$LOGDIR" # #----------------------------------------------------------------------- # @@ -2257,7 +2286,7 @@ check_var_valid_value "WRITE_DOPOST" "valid_vals_WRITE_DOPOST" # Set WRITE_DOPOST to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -WRITE_DOPOST=${WRITE_DOPOST^^} +WRITE_DOPOST=$(echo_uppercase $WRITE_DOPOST) if [ "$WRITE_DOPOST" = "TRUE" ] || \ [ "$WRITE_DOPOST" = "YES" ]; then WRITE_DOPOST="TRUE" @@ -2287,7 +2316,7 @@ check_var_valid_value "QUILTING" "valid_vals_QUILTING" # Set QUILTING to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -QUILTING=${QUILTING^^} +QUILTING=$(echo_uppercase $QUILTING) if [ "$QUILTING" = "TRUE" ] || \ [ "$QUILTING" = "YES" ]; then QUILTING="TRUE" @@ -2307,7 +2336,7 @@ check_var_valid_value "PRINT_ESMF" "valid_vals_PRINT_ESMF" # Set PRINT_ESMF to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -PRINT_ESMF=${PRINT_ESMF^^} +PRINT_ESMF=$(echo_uppercase $PRINT_ESMF) if [ "${PRINT_ESMF}" = "TRUE" ] || \ [ "${PRINT_ESMF}" = "YES" ]; then PRINT_ESMF="TRUE" @@ -2458,7 +2487,7 @@ cp_vrfy $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ${GLOBAL_VAR_DEFNS_FP} # # Read all lines of GLOBAL_VAR_DEFNS file into the variable line_list. -line_list=$( sed -r -e "s/(.*)/\1/g" ${GLOBAL_VAR_DEFNS_FP} ) +line_list=$( $SED -r -e "s/(.*)/\1/g" ${GLOBAL_VAR_DEFNS_FP} ) # # Loop through the lines in line_list and concatenate lines ending with # the line bash continuation character "\". @@ -2499,7 +2528,7 @@ done <<< "${line_list}" #----------------------------------------------------------------------- # # Also should remove trailing whitespace... -line_list=$( sed -r \ +line_list=$( $SED -r \ -e "s/^([ ]*)([^ ]+.*)/\2/g" \ -e "/^#.*/d" \ -e "/^$/d" \ @@ -2545,7 +2574,7 @@ str_to_insert=${str_to_insert//$'\n'/\\n} # the string "#!", e.g. "#!/bin/bash"). # regexp="(^#!.*)" -sed -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} +$SED -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} # # Loop through the lines in line_list. # @@ -2559,7 +2588,7 @@ while read crnt_line; do # or more characters representing the value that the variable is being # set to. # - var_name=$( printf "%s" "${crnt_line}" | sed -n -r -e "s/^([^ ]*)=.*/\1/p" ) + var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) #echo #echo "============================" #printf "%s\n" "var_name = \"${var_name}\"" @@ -2766,7 +2795,7 @@ FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) GLOBAL_VAR_DEFNS_FP="${GLOBAL_VAR_DEFNS_FP}" # Try this at some point instead of hard-coding it as above; it's a more # flexible approach (if it works). -#GLOBAL_VAR_DEFNS_FP=$( readlink -f "${BASH_SOURCE[0]}" ) +#GLOBAL_VAR_DEFNS_FP=$( $READLINK -f "${BASH_SOURCE[0]}" ) DATA_TABLE_TMPL_FN="${DATA_TABLE_TMPL_FN}" DIAG_TABLE_TMPL_FN="${DIAG_TABLE_TMPL_FN}" @@ -3011,6 +3040,7 @@ FVCOM_FILE="${FVCOM_FILE}" # NCORES_PER_NODE="${NCORES_PER_NODE}" PE_MEMBER01="${PE_MEMBER01}" +RUN_CMD_FCST="${RUN_CMD_FCST}" # #----------------------------------------------------------------------- # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index bc6a2f0832..375543d354 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -8,7 +8,11 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + if [[ $(uname -s) == Darwin ]]; then + local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) + else + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + fi local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -48,6 +52,15 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # +# Source the file that defines MacOS-specific UNIX command-line +# utilities, that mimic the functionality of the GNU equivalents +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/define_macos_utilities.sh +# +#----------------------------------------------------------------------- +# # Source the file containing the functions that print out messages. # #----------------------------------------------------------------------- @@ -82,6 +95,15 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # +# Source the file containing the functions that will echo given strings +# as uppercase or lowercase +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/change_case.sh +# +#----------------------------------------------------------------------- +# # Source the file containing the function that checks for preexisting # directories or files and handles them according to a specified method # (which can be one of "delete", "rename", and "quit"). diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index d5eb39b9b7..f42093fa64 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -3,9 +3,10 @@ # valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE") +valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") +valid_vals_WORKFLOW_MANAGER=("rocoto" "none") valid_vals_PREDEF_GRID_NAME=( \ "RRFS_CONUS_25km" \ "RRFS_CONUS_13km" \ From 9490f2f498958cbde0e929eb92ece5d26dea30d3 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 23 Sep 2021 06:18:36 -0400 Subject: [PATCH 088/203] Add option for fv3gfs_aqm to run_post (#585) * Add option for fv3gfs_aqm * Change parameter name --- scripts/exregional_run_post.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index b944ea169c..9b0f26a42f 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -182,7 +182,11 @@ to the temporary work directory (tmp_dir): tmp_dir = \"${tmp_dir}\" ====================================================================" else - post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam.txt" + if [ ${FCST_MODEL} = "fv3gfs_aqm" ]; then + post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam_cmaq.txt" + else + post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam.txt" + fi print_info_msg " ==================================================================== Copying the default post flat file specified by post_config_fp to the @@ -260,6 +264,11 @@ post_mn=${post_time:10:2} # # Create the input text file to the post-processor executable. # +if [ ${FCST_MODEL} = "fv3gfs_aqm" ]; then + post_itag_add="aqfcmaq_on=.true.," +else + post_itag_add="" +fi cat > itag < Date: Tue, 28 Sep 2021 08:41:17 -0600 Subject: [PATCH 089/203] Remove Julie from CODEOWNERS (#609) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 40196de744..c05ae70791 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @JulieSchramm @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA +* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners From 2a3a71299eed0340ba43d567a8298e3d1ebc6f5a Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 6 Oct 2021 13:16:51 -0600 Subject: [PATCH 090/203] Add do_gsl* namelist entries to YAML file for FV3_HRRR SDF. (#597) --- ush/templates/FV3.input.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 4d7c99b541..8e9b43fe5d 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -70,6 +70,9 @@ FV3_HRRR: do_mynnsfclay: True do_sfcperts: !!python/none gwd_opt: 3 + do_gsl_drag_ss: True + do_gsl_drag_tofd: True + do_gsl_drag_ls_bl: True iaer: 5111 icliq_sw: 2 imfdeepcnv: -1 From f0a1dacf4259dbc3258682e4e9ac1594e4efc4fc Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 7 Oct 2021 06:22:14 -0400 Subject: [PATCH 091/203] Fix unbound SED for cron commands in the launch script (#606) * Add source_util_funcs to launch script --- ush/launch_FV3LAM_wflow.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 72cdd74c77..fb2c71c965 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -25,6 +25,10 @@ else fi scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) + +ushdir="${scrfunc_dir}" +. $ushdir/source_util_funcs.sh + # #----------------------------------------------------------------------- # @@ -309,8 +313,8 @@ while read -r line; do # if [ $i -gt 0 ]; then im1=$((i-1)) - cycle_str[im1]=$( echo "$line" | sed -r -n -e "s/${regex_search}/\1/p" ) - cycle_status[im1]=$( echo "$line" | sed -r -n -e "s/${regex_search}/\2/p" ) + cycle_str[im1]=$( echo "$line" | $SED -r -n -e "s/${regex_search}/\1/p" ) + cycle_status[im1]=$( echo "$line" | $SED -r -n -e "s/${regex_search}/\2/p" ) fi i=$((i+1)) done <<< "${rocotostat_output}" From e5e7e293516aa8b7b678abcfb4b03656c31273c6 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 19 Oct 2021 17:12:11 -0600 Subject: [PATCH 092/203] Adding an environment file for installing pygraf. (#604) --- ush/Python/environment.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 ush/Python/environment.yml diff --git a/ush/Python/environment.yml b/ush/Python/environment.yml new file mode 100644 index 0000000000..8e08987ce0 --- /dev/null +++ b/ush/Python/environment.yml @@ -0,0 +1,15 @@ +name: pygraf +channels: + - conda-forge + - defaults +dependencies: + - python=3.7* + - basemap=1.2* + - basemap-data-hires=1.2.1 + - pynio=1.5.5 + - matplotlib=3.2* + - metpy=0.12.1 + - pylint=2.4* + - pytest=6.1* + - pyyaml=5.3.1 + - xarray=0.15.1 From c203a387be88722d08416fcb913c079aa60807ba Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 22 Oct 2021 07:56:40 -0400 Subject: [PATCH 093/203] Modify workflow environment for orion (#616) --- modulefiles/tasks/orion/make_grid.local | 3 ++- modulefiles/tasks/orion/make_ics.local | 2 +- modulefiles/tasks/orion/make_lbcs.local | 2 +- modulefiles/tasks/orion/run_fcst.local | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 93a76f6ca0..4de2a79ca9 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,5 +1,6 @@ #%Module + module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3 +module load miniconda3/3.8 setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 93a76f6ca0..3703a9ba17 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,5 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3 +module load miniconda3/3.8 setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 93a76f6ca0..3703a9ba17 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,5 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3 +module load miniconda3/3.8 setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 93a76f6ca0..3703a9ba17 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,5 +1,5 @@ #%Module module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3 +module load miniconda3/3.8 setenv SRW_ENV regional_workflow From ca6113895086eb41cfb4a2841b96ccf55f4807b0 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 27 Oct 2021 06:34:20 -0400 Subject: [PATCH 094/203] increase wtime_make_orog for RRFS NA 3km on WCOSS (#622) * increase wtime_make_orog for RRFS_NA_3km on wcoss * remove ppn_run_fcst from WE2E script for RRFS_NA_3km * Add EMC configuration in comment --- ...d_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh | 2 -- ush/config_defaults.sh | 2 +- ush/set_predef_grid_params.sh | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index 9b4e0da931..a01a308072 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -45,8 +45,6 @@ PPN_MAKE_ICS="4" PPN_MAKE_LBCS="4" WTIME_MAKE_LBCS="01:00:00" -PPN_RUN_FCST="24" - NNODES_RUN_POST="6" PPN_RUN_POST="12" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index dd6a2968c7..35c744039f 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1515,7 +1515,7 @@ PPN_VX_ENSPOINT_PROB="1" # Walltimes. # WTIME_MAKE_GRID="00:20:00" -WTIME_MAKE_OROG="00:20:00" +WTIME_MAKE_OROG="01:00:00" WTIME_MAKE_SFC_CLIMO="00:20:00" WTIME_GET_EXTRN_ICS="00:45:00" WTIME_GET_EXTRN_LBCS="00:45:00" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index aca2a26e36..dd664670ab 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1227,8 +1227,8 @@ case ${PREDEF_GRID_NAME} in DT_ATMOS="${DT_ATMOS:-36}" - LAYOUT_X="${LAYOUT_X:-18}" - LAYOUT_Y="${LAYOUT_Y:-36}" + LAYOUT_X="${LAYOUT_X:-18}" # 40 - EMC operational configuration + LAYOUT_Y="${LAYOUT_Y:-36}" # 45 - EMC operational configuration BLOCKSIZE="${BLOCKSIZE:-28}" if [ "$QUILTING" = "TRUE" ]; then From c236f520c3a142a40ae13e076077237f1ce05ad6 Mon Sep 17 00:00:00 2001 From: David Wright Date: Wed, 27 Oct 2021 18:12:16 -0400 Subject: [PATCH 095/203] Allow FVCOM to be used in Warm and Cold Starts (#620) * Change fvcom_to_FV3 use to allow for warm/cold start sfc data and time selection in a FVCOM file * Update valid values for FVCOM_WCSTART --- scripts/exregional_make_ics.sh | 6 +++++- ush/config_defaults.sh | 7 +++++++ ush/setup.sh | 4 ++++ ush/valid_param_vals.sh | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index ec0fee623c..951477c93d 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -658,8 +658,10 @@ mv_vrfy gfs.bndy.nc ${ics_dir}/gfs_bndy.tile${TILE_RGNL}.000.nc # if [ "${USE_FVCOM}" = "TRUE" ]; then +#Format for fvcom_time: YYYY-MM-DDTHH:00:00.000000 fvcom_exec_fn="fvcom_to_FV3" fvcom_exec_fp="$EXECDIR/${fvcom_exec_fn}" + fvcom_time="${DATE_FIRST_CYCL:0:4}-${DATE_FIRST_CYCL:4:2}-${DATE_FIRST_CYCL:6:2}T${CYCL_HRS[0]}:00:00.000000" if [ ! -f "${fvcom_exec_fp}" ]; then print_err_msg_exit "\ The executable (fvcom_exec_fp) for processing FVCOM data onto FV3-LAM @@ -680,13 +682,15 @@ Please check the following user defined variables: cp_vrfy ${fvcom_data_fp} ${ics_dir}/fvcom.nc cd_vrfy ${ics_dir} - ${APRUN} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc || \ + ${APRUN} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \ print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" The following variables were being used: FVCOM_DIR = \"${FVCOM_DIR}\" FVCOM_FILE = \"${FVCOM_FILE}\" + fvcom_time = \"${fvcom_time}\" + FVCOM_WCSTART = \"${FVCOM_WCSTART}\" ics_dir = \"${ics_dir}\" fvcom_exe_dir = \"${fvcom_exe_dir}\" fvcom_exe = \"${fvcom_exe}\"" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 35c744039f..1ce86b9b12 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1723,6 +1723,12 @@ HALO_BLEND="10" # FV3-LAM grid. This flag will be used in make_ics to modify sfc_data.nc # after chgres_cube is run by running the routine process_FVCOM.exe # +# FVCOM_WCSTART: +# Define if this is a "warm" start or a "cold" start. Setting this to +# "warm" will read in sfc_data.nc generated in a RESTART directory. +# Setting this to "cold" will read in the sfc_data.nc generated from +# chgres_cube in the make_ics portion of the workflow. +# # FVCOM_DIR: # User defined directory where FVCOM data already interpolated to FV3-LAM # grid is located. File name in this path should be "fvcom.nc" to allow @@ -1735,6 +1741,7 @@ HALO_BLEND="10" #------------------------------------------------------------------------ # USE_FVCOM="FALSE" +FVCOM_WCSTART="cold" FVCOM_DIR="/user/defined/dir/to/fvcom/data" FVCOM_FILE="fvcom.nc" # diff --git a/ush/setup.sh b/ush/setup.sh index 96b1ef0717..a166fafccd 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -450,10 +450,13 @@ fi # # Make sure that USE_FVCOM is set to a valid value and assign directory # and file names. +# +# Make sure that FVCOM_WCSTART is set to lowercase "warm" or "cold" # #----------------------------------------------------------------------- # check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" +check_var_valid_value "FVCOM_WCSTART" "valid_vals_FVCOM_WCSTART" # # Set USE_FVCOM to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. @@ -466,6 +469,7 @@ elif [ "$USE_FVCOM" = "FALSE" ] || \ [ "$USE_FVCOM" = "NO" ]; then USE_FVCOM="FALSE" fi +FVCOM_WCSTART=$(echo_lowercase $FVCOM_WCSTART) # #----------------------------------------------------------------------- # diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index f42093fa64..1b3acf00ae 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -72,6 +72,7 @@ valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_FVCOM_WCSTART=("warm" "WARM" "cold" "COLD") valid_vals_COMPILER=("intel" "gnu") valid_vals_SUB_HOURLY_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") From be1d0f9c9767bd7b1789e12cf9ddf32652e3e410 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Wed, 10 Nov 2021 17:12:02 -0700 Subject: [PATCH 096/203] Add support for Rocoto with generic LINUX platform (#617) * Adding all changes. Untested at this point. * Chris's additions for running on AWS. * Changes needed for AWS platform. * Clean up comment to match code. * Fix spelling error. * More docs for choosing linux + rocoto. * Require NCORES_PER_NODE for all machines. * Ensure variables are expanded within RUN_CMD --- scripts/exregional_make_ics.sh | 1 + scripts/exregional_make_lbcs.sh | 1 + scripts/exregional_make_orog.sh | 2 + scripts/exregional_run_fcst.sh | 2 + ush/config_defaults.sh | 33 +++++++++- ush/launch_FV3LAM_wflow.sh | 54 +++-------------- ush/load_modules_run_task.sh | 104 ++++++++++---------------------- ush/setup.sh | 58 ++++++++++++------ ush/templates/FV3LAM_wflow.xml | 2 +- 9 files changed, 119 insertions(+), 138 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 951477c93d..dd64a4999a 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -131,6 +131,7 @@ case "$MACHINE" in ;; "LINUX") + ulimit -s unlimited APRUN=$RUN_CMD_UTILS ;; diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 1b1f84ff88..01aeaed965 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -131,6 +131,7 @@ case "$MACHINE" in ;; "LINUX") + ulimit -s unlimited APRUN=$RUN_CMD_UTILS ;; diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index e90b3b762c..afd40d74d2 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -153,6 +153,8 @@ case "$MACHINE" in "LINUX") APRUN=time + ulimit -s unlimited + ulimit -a ;; *) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 4b16f79d95..aae18a749a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -159,6 +159,8 @@ case "$MACHINE" in ;; "LINUX") + ulimit -s unlimited + ulimit -a APRUN=$RUN_CMD_FCST ;; diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 1ce86b9b12..ba3711f906 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -41,7 +41,12 @@ RUN_ENVIR="nco" # Set machine and queue parameters. Definitions: # # MACHINE: -# Machine on which the workflow will run. +# Machine on which the workflow will run. If you are NOT on a named, +# supported platform, and you want to use the Rocoto workflow manager, +# you will need set MACHINE="linux" and WORKFLOW_MANAGER="rocoto". This +# combination will assume a Slurm batch manager when generating the XML. +# Please see ush/valid_param_vals.sh for a full list of supported +# platforms. # # ACCOUNT: # The account under which to submit jobs to the queue. @@ -49,7 +54,27 @@ RUN_ENVIR="nco" # WORKFLOW_MANAGER: # The workflow manager to use (e.g. rocoto). This is set to "none" by # default, but if the machine name is set to a platform that supports -# rocoto, this will be overwritten and set to "rocoto". +# rocoto, this will be overwritten and set to "rocoto". If set +# explicitly to rocoto along with the use of the MACHINE=linux target, +# the configuration layer assumes a Slurm batch manager when generating +# the XML. Valid options: "rocoto" or "none" +# +# NCORES_PER_NODE: +# The number of cores available per node on the compute platform. Set +# for supported platforms in setup.sh, but is now also configurable for +# all platforms. +# +# LMOD_PATH: +# Path to the LMOD sh file on your Linux system. Is set automatically +# for supported machines. +# +# BUILD_ENV_FN: +# Name of alternative build environment file to use if using an +# unsupported platform. Is set automatically for supported machines. +# +# WFLOW_ENV_FN: +# Name of alternative workflow environment file to use if using an +# unsupported platform. Is set automatically for supported machines. # # SCHED: # The job scheduler to use (e.g. slurm). Set this to an empty string in @@ -109,6 +134,10 @@ RUN_ENVIR="nco" MACHINE="BIG_COMPUTER" ACCOUNT="project_name" WORKFLOW_MANAGER="none" +NCORES_PER_NODE="" +LMOD_PATH="" +BUILD_ENV_FN="" +WFLOW_ENV_FN="" SCHED="" PARTITION_DEFAULT="" QUEUE_DEFAULT="" diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index fb2c71c965..fe7ec8df67 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -88,6 +88,7 @@ fi #----------------------------------------------------------------------- # . $exptdir/var_defns.sh +. ${USHDIR}/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -105,13 +106,7 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -if [ "$MACHINE" = "CHEYENNE" ]; then - module use -a /glade/p/ral/jntp/UFS_SRW_app/modules/ - module load rocoto -elif [ "$MACHINE" = "ORION" ]; then - module purge - module load contrib rocoto -elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then +if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then module purge module load lsf/10.1 module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ @@ -122,8 +117,14 @@ elif [ "$MACHINE" = "WCOSS_CRAY" ]; then module use -a /usrx/local/emc_rocoto/modulefiles module load rocoto/1.3.0rc2 else + machine=$(echo_lowercase $MACHINE) + env_fn=${WFLOW_ENV_FN:-"wflow_${machine}.env"} + env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" module purge - module load rocoto + source "${env_fp}" || print_err_msg_exit "\ + Sourcing platform-specific environment file (env_fp) for + the workflow task failed : + env_fp = \"${env_fp}\"" fi # #----------------------------------------------------------------------- @@ -166,35 +167,7 @@ cd "$exptdir" #----------------------------------------------------------------------- # -#rocotorun_output=$( ls -alF ) -#echo -#echo "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" -#echo "${rocotorun_output}" -#echo "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" - -#rocotorun_output=$( \ -#rocotorun -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 \ -#) -#rocotorun_output=$( (rocotorun -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10) 2>&1 ) # This freezes the script. -#rocotorun_output=$( (rocotorun -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10) 1>&2 ) # This leaves rocotorun_output empty. -#rocotorun_output=$( rocotorun -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 ) -#{ error=$(command 2>&1 1>&$out); } {out}>&1 -#{ rocotorun_output=$( rocotorun -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 2>&1 1>&$out); } {out}>&1 # This freezes the script. - -# -# Ideally, the following two lines should work, but for some reason the -# output of rocotorun cannot be captured in a variable using the $(...) -# notation. Maybe it's not being written to stdout, although I tried -# redirecting stderr to stdout and other tricks but nothing seemed to -# work. For this reason, below we first redirect the output of rocoto- -# run to a temporary file and then read in the contents of that file in- -# to the rocotorun_output variable using the cat command. -# -#rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" -#rocotorun_output=$( eval ${rocotorun_cmd} 2>&1 ) -# tmp_fn="rocotorun_output.txt" -#rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10 > ${tmp_fn}" rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 rocotorun_output=$( cat "${tmp_fn}" ) @@ -221,18 +194,9 @@ done <<< "${rocotorun_output}" # #----------------------------------------------------------------------- # -#rocotostat_cmd="{ pwd; rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10; }" -#rocotostat_cmd="{ pwd; ls -alF; rocotostat -w ${WFLOW_XML_FN} -d ${rocoto_database_fn} -v 10; }" -#rocotostat_cmd="{ pwd; ls -alF; rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10; }" -#rocotostat_cmd="{ pwd; rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10; }" -#rocotostat_cmd="{ rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10; }" rocotostat_cmd="rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" -#rocotostat_output=$( pwd; rocotostat -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 2>&1 ) -#rocotostat_output=$( rocotostat -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 2>&1 ) rocotostat_output=$( eval ${rocotostat_cmd} 2>&1 ) -#rocotostat_output=$( ${rocotostat_cmd} 2>&1 ) -#rocotostat_output=$( { pwd; ls -alF; } 2>&1 ) error_msg="DEAD" while read -r line; do grep_output=$( printf "$line" | grep "${error_msg}" ) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 411f0bdb1e..1f0d881260 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -71,30 +71,6 @@ fi # ..." and "module load ..." calls later below that are used to load the # appropriate module file for the specified task. # -# Note that the build of the FV3 forecast model code generates the shell -# script at -# -# ${UFS_WTHR_MDL_DIR}/NEMS/src/conf/module-setup.sh -# -# that can be used to initialize the Lmod (Lua-based module) system/ -# software for handling modules. This script: -# -# 1) Detects the shell in which it is being invoked (i.e. the shell of -# the "parent" script in which it is being sourced). -# 2) Detects the machine it is running on and and calls the appropriate -# (shell- and machine-dependent) initalization script to initialize -# Lmod. -# 3) Purges all modules. -# 4) Uses the "module use ..." command to prepend or append paths to -# Lmod's search path (MODULEPATH). -# -# We could use this module-setup.sh script to initialize Lmod, but since -# it is only found in the forecast model's directory tree, here we pre- -# fer to perform our own initialization. Ideally, there should be one -# module-setup.sh script that is used by all external repos/codes, but -# such a script does not exist. If/when it does, we will consider -# switching to it instead of using the case-statement below. -# #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " @@ -128,10 +104,14 @@ case "$MACHINE" in ;; # *) - print_err_msg_exit "\ -The script to source to initialize lmod (module loads) has not yet been -specified for the current machine (MACHINE): - MACHINE = \"$MACHINE\"" + if [[ -n ${LMOD_PATH:-""} && -f ${LMOD_PATH:-""} ]] ; then + . ${LMOD_PATH} + else + print_err_msg_exit "\ + The script to source to initialize lmod (module loads) has not yet been + specified for the current machine (MACHINE): + MACHINE = \"$MACHINE\"" + fi ;; # esac @@ -147,13 +127,15 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # -# Sourcing ufs-srweather-app README file (in directory specified by mod- -# ules_dir) for the specified task +# Sourcing ufs-srweather-app build env file # #----------------------------------------------------------------------- # + +module purge + machine=$(echo_lowercase $MACHINE) -env_fn="build_${machine}_${COMPILER}.env" +env_fn=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" source "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the @@ -172,25 +154,23 @@ workflow task specified by task_name failed: # sets environment variables (including prepending/appending to paths) # and loads modules. # -# The regional_workflow repository contains module files for all the +# The regional_workflow repository contains module files for the # workflow tasks in the template rocoto XML file for the FV3-LAM work- -# flow. The full path to a module file for a given task is +# flow that need modules not loaded in the env_fn above. +# +# The full path to a module file for a given task is # -# $HOMErrfs/modulefiles/$machine/${task_name} +# $HOMErrfs/modulefiles/$machine/${task_name}.local # # where HOMErrfs is the base directory of the workflow, machine is the # name of the machine that we're running on (in lowercase), and task_- -# name is the name of the current task (an input to this script). The -# collection of modulefiles is staged by the generate_workflow.sh -# script. Please see that script for information on their creation. +# name is the name of the current task (an input to this script). # #----------------------------------------------------------------------- # modules_dir="$HOMErrfs/modulefiles/tasks/$machine" modulefile_name="${task_name}" default_modules_dir="$HOMErrfs/modulefiles" -default_modulefile_name="${machine}.default" -use_default_modulefile=0 # #----------------------------------------------------------------------- # @@ -198,46 +178,28 @@ use_default_modulefile=0 # #----------------------------------------------------------------------- # - print_info_msg "$VERBOSE" " + +print_info_msg "$VERBOSE" " Loading modules for task \"${task_name}\" ..." - module use "${modules_dir}" || print_err_msg_exit "\ +module use "${modules_dir}" || print_err_msg_exit "\ Call to \"module use\" command failed." - # - # If NOT using the default modulefile... - # -# if [ ${use_default_modulefile} -eq 0 ]; then -# -# module use -a "${modules_dir}" || print_err_msg_exit "\ -#Call to \"module use\" command failed." -# - # - # Load the .local module file if available for the given task - # - modulefile_local="${task_name}.local" - if [ -f ${modules_dir}/${modulefile_local} ]; then - module load "${modulefile_local}" || print_err_msg_exit "\ -Loading .local module file (in directory specified by mod- -ules_dir) for the specified task (task_name) failed: - task_name = \"${task_name}\" - modulefile_local = \"${modulefile_local}\" - modules_dir = \"${modules_dir}\"" - fi - -# else # using default modulefile # -# module load "${default_modulefile_name}" || print_err_msg_exit "\ -#Loading of default module file failed: -# task_name = \"${task_name}\" -# default_modulefile_name = \"${default_modulefile_name}\" -# default_modules_dir = \"${default_modules_dir}\"" +# Load the .local module file if available for the given task # -# fi +modulefile_local="${task_name}.local" +if [ -f ${modules_dir}/${modulefile_local} ]; then + module load "${modulefile_local}" || print_err_msg_exit "\ + Loading .local module file (in directory specified by mod- + ules_dir) for the specified task (task_name) failed: + task_name = \"${task_name}\" + modulefile_local = \"${modulefile_local}\" + modules_dir = \"${modules_dir}\"" +fi - module list +module list -#fi #End if statement for tasks that load no modules # Modules that use conda and need an environment activated will set the # SRW_ENV variable to the name of the environment to be activated. That diff --git a/ush/setup.sh b/ush/setup.sh index a166fafccd..1dcc748abc 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1,3 +1,4 @@ +#!/bin/bash # #----------------------------------------------------------------------- # @@ -516,7 +517,7 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" # several queues. These queues are defined in the default and local # workflow/experiment configuration script. # -# Also, set the machine-dependent flag RELAITVE_OR_NULL that specifies +# Also, set the machine-dependent flag RELATIVE_OR_NULL that specifies # the flag to pass to the link creation command (ln_vrfy) when attempting # to create relative symlinks. On machines that don't support relative # symlinks, it should be set to a null string. @@ -524,12 +525,11 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" #----------------------------------------------------------------------- # RELATIVE_LINK_FLAG="" -NCORES_PER_NODE="2" # Need some arbitrary default value to avoid division by zero errors case $MACHINE in "WCOSS_CRAY") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="24" + NCORES_PER_NODE="${NCORES_PER_NODE:-24}" SCHED="lsfcray" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} @@ -540,7 +540,7 @@ case $MACHINE in "WCOSS_DELL_P3") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=24 + NCORES_PER_NODE="${NCORES_PER_NODE:-24}" SCHED="lsf" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} @@ -551,7 +551,7 @@ case $MACHINE in "HERA") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=40 + NCORES_PER_NODE="${NCORES_PER_NODE:-40}" SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} @@ -565,7 +565,7 @@ case $MACHINE in "ORION") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=40 + NCORES_PER_NODE="${NCORES_PER_NODE:-40}" SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} @@ -579,7 +579,7 @@ case $MACHINE in "JET") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=24 + NCORES_PER_NODE="${NCORES_PER_NODE:-24}" SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} @@ -593,7 +593,7 @@ case $MACHINE in "ODIN") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=24 + NCORES_PER_NODE="${NCORES_PER_NODE:-24}" SCHED=${SCHED:-"slurm"} PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} @@ -607,7 +607,7 @@ case $MACHINE in "CHEYENNE") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=36 + NCORES_PER_NODE="${NCORES_PER_NODE:-36}" SCHED=${SCHED:-"pbspro"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} QUEUE_HPSS=${QUEUE_HPSS:-"regular"} @@ -618,7 +618,7 @@ case $MACHINE in "STAMPEDE") WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE=68 + NCORES_PER_NODE="${NCORES_PER_NODE:-68}" SCHED="slurm" PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} @@ -636,11 +636,26 @@ case $MACHINE in ;; "LINUX") - WORKFLOW_MANAGER="none" - SCHED="none" + WORKFLOW_MANAGER=${WORKFLOW_MANAGER:-"none"} + SCHED=${SCHED:-"none"} + ;; + + "*") + NCORES_PER_NODE="2" # Need some arbitrary default value to avoid division by zero errors + + print_err_msg_exit "\ + You are running on an unknown platform! MACHINE=${MACHINE} is not a valid + choice." ;; esac + +if [ -z "$NCORES_PER_NODE" ]; then + print_err_msg_exit "\ + NCORES_PER_NODE is a required setting for your platform! Please + set it in config.sh. + MACHINE = ${MACHINE}" +fi # #----------------------------------------------------------------------- # @@ -662,9 +677,8 @@ check_var_valid_value "SCHED" "valid_vals_SCHED" # #----------------------------------------------------------------------- # -# If we are using a workflow manager, run some checks. First, -# verify that the ACCOUNT variable is not empty. Second, ensure that the -# custom RUN_CMD variables are not set. +# If we are using a workflow manager check that the ACCOUNT variable is +# not empty. # #----------------------------------------------------------------------- # @@ -675,9 +689,6 @@ The variable ACCOUNT cannot be empty if you are using a workflow manager: ACCOUNT = \"$ACCOUNT\" WORKFLOW_MANAGER = \"$WORKFLOW_MANAGER\"" fi - RUN_CMD_UTILS="" - RUN_CMD_FCST="" - RUN_CMD_POST="" fi # #----------------------------------------------------------------------- @@ -2946,6 +2957,15 @@ fi # #----------------------------------------------------------------------- # +# Because RUN_CMD_FCST can include PE_MEMBER01 (and theoretically other +# variables calculated in this script), delete the first occurrence of it +# in the var_defns file, and write it again at the end. +# +#----------------------------------------------------------------------- +$SED -i '/^RUN_CMD_FCST=/d' $GLOBAL_VAR_DEFNS_FP +# +#----------------------------------------------------------------------- +# # Continue appending variable definitions to the variable definitions # file. # @@ -3044,7 +3064,7 @@ FVCOM_FILE="${FVCOM_FILE}" # NCORES_PER_NODE="${NCORES_PER_NODE}" PE_MEMBER01="${PE_MEMBER01}" -RUN_CMD_FCST="${RUN_CMD_FCST}" +RUN_CMD_FCST=$(eval echo ${RUN_CMD_FCST}) # #----------------------------------------------------------------------- # diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index f7e7b94715..cc77b5ad40 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -414,7 +414,7 @@ MODULES_RUN_TASK_FP script. &RSRV_FCST; &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" - {%- if machine in ["JET", "HERA"] %} + {%- if machine in ["JET", "HERA", "LINUX"] %} {{ ncores_run_fcst }} {{ native_run_fcst }} {%- else %} From f014ed91295e239f7444ccc88a6a6171f9b2d0ea Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 11 Nov 2021 11:40:37 -0500 Subject: [PATCH 097/203] Add ulimit to the task scripts for Orion (#626) * Add ulimit to make_grid script for orion * change stacksize of make_ics * Add ulimit to run_post script * Remove change for Hera --- scripts/exregional_make_grid.sh | 2 ++ scripts/exregional_make_ics.sh | 1 + scripts/exregional_run_post.sh | 2 ++ ...rid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh | 1 + 4 files changed, 6 insertions(+) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 9d4a1d9f6d..2b876401dc 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -111,6 +111,8 @@ case "$MACHINE" in ;; "ORION") + ulimit -s unlimited + ulimit -a APRUN="time" ;; diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index dd64a4999a..c82a987add 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -105,6 +105,7 @@ case "$MACHINE" in "ORION") ulimit -s unlimited + ulimit -a APRUN="srun" ;; diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 9b0f26a42f..2632d7d55e 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -125,6 +125,8 @@ case "$MACHINE" in ;; "ORION") + ulimit -s unlimited + ulimit -a APRUN="srun" ;; diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index a01a308072..c088805dec 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -48,6 +48,7 @@ WTIME_MAKE_LBCS="01:00:00" NNODES_RUN_POST="6" PPN_RUN_POST="12" +OMP_STACKSIZE_MAKE_ICS="2048m" OMP_STACKSIZE_RUN_FCST="2048m" ############################################################################### From 302c2116f3eba4b959a77057c07ef854452b8232 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 11 Nov 2021 13:22:23 -0500 Subject: [PATCH 098/203] Modify WE2E tests for NCO mode (#629) * Modify nco cases of WE2E * Add nco we2e tests for v16 and hrrr suites --- ...id_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 26 ------------------- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 26 +++++++++++++++++++ ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 26 +++++++++++++++++++ ...25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh} | 6 ++--- ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 25 ------------------ ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 7 +++-- ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} | 15 ++++++----- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 25 ------------------ ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 25 ------------------ 9 files changed, 66 insertions(+), 115 deletions(-) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/{config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh => config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh} (75%) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/{config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh => config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} (52%) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh deleted file mode 100644 index 3431f7cdc3..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the CONUS_25km_GFDLgrid grid (which is a GFDLgrid type -# of grid) using the GFS_v16 physics suite with ICs and LBCs derived from -# the FV3GFS. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" -CCPP_PHYS_SUITE="FV3_GFS_v16" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190901" -CYCL_HRS=( "18" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..141a9857ce --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_13km grid using the GFS_v15p2 physics +# suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh new file mode 100644 index 0000000000..1b7d92162c --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in nco mode completes +# successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 physics +# suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="nco" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh similarity index 75% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 890395ba8c..656a4d0da9 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -3,15 +3,15 @@ # ------------------------ # # This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_3km grid using the HRRR physics suite +# successfully on the RRFS_CONUS_25km grid using the FV3_GFS_v15p2 physics suite # with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_HRRR" +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 9bd42fa735..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_25km grid using the GSD_SAR physics -# suite with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index b0d240892d..aac14a4794 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -17,11 +17,10 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190901" -CYCL_HRS=( "18" ) +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 52% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index d8eeef6c5c..64c267636f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -3,23 +3,24 @@ # ------------------------ # # This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_3km grid using the GFS_2017_gfdlmp_regional -# physics suite with ICs and LBCs derived from the FV3GFS. +# successfully on the RRFS_CONUS_3km grid using the GFS_v16 physics +# suite with ICs and LBCs derived from the FV3GFS. # RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" +CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" +FV3GFS_FILE_FMT_LBCS="grib2" -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190901" -CYCL_HRS=( "18" ) +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index eae6b7401a..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_3km grid using the GSD_SAR physics -# suite with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 3770547eff..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_SUBCONUS_3km grid using the GSD_SAR physics -# suite with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" From 2c0e9e5634834828d3cf846457650d8af4c72b45 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 15 Nov 2021 17:17:57 -0500 Subject: [PATCH 099/203] Remove rocoto for wcoss from launch script (#636) --- ush/launch_FV3LAM_wflow.sh | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index fe7ec8df67..76c964083a 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -106,26 +106,14 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - module purge - module load lsf/10.1 - module use /gpfs/dell3/usrx/local/dev/emc_rocoto/modulefiles/ - module load ruby/2.5.1 rocoto/1.3.0rc2 -elif [ "$MACHINE" = "WCOSS_CRAY" ]; then - module purge - module load xt-lsfhpc/9.1.3 - module use -a /usrx/local/emc_rocoto/modulefiles - module load rocoto/1.3.0rc2 -else - machine=$(echo_lowercase $MACHINE) - env_fn=${WFLOW_ENV_FN:-"wflow_${machine}.env"} - env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" - module purge - source "${env_fp}" || print_err_msg_exit "\ - Sourcing platform-specific environment file (env_fp) for - the workflow task failed : - env_fp = \"${env_fp}\"" -fi +machine=$(echo_lowercase $MACHINE) +env_fn=${WFLOW_ENV_FN:-"wflow_${machine}.env"} +env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" +module purge +source "${env_fp}" || print_err_msg_exit "\ + Sourcing platform-specific environment file (env_fp) for +the workflow task failed : +env_fp = \"${env_fp}\"" # #----------------------------------------------------------------------- # From 75ae0f682f11a924a0b78d4ca9e3869b23f87fe8 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 17 Nov 2021 12:51:55 -0500 Subject: [PATCH 100/203] Rename unchanged readlink (#638) --- scripts/exregional_run_ensgridvx.sh | 2 +- scripts/exregional_run_ensgridvx_mean.sh | 2 +- scripts/exregional_run_ensgridvx_prob.sh | 2 +- scripts/exregional_run_enspointvx.sh | 2 +- scripts/exregional_run_enspointvx_mean.sh | 2 +- scripts/exregional_run_enspointvx_prob.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh index d00d0f0fe1..78cb5c2ba6 100755 --- a/scripts/exregional_run_ensgridvx.sh +++ b/scripts/exregional_run_ensgridvx.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh index 7629cd6691..b255adb868 100755 --- a/scripts/exregional_run_ensgridvx_mean.sh +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh index fc4749d15c..2ed9bdfead 100755 --- a/scripts/exregional_run_ensgridvx_prob.sh +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh index e3985314de..571711bf56 100755 --- a/scripts/exregional_run_enspointvx.sh +++ b/scripts/exregional_run_enspointvx.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh index cd0d3f6367..4cd6039b9b 100755 --- a/scripts/exregional_run_enspointvx_mean.sh +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh index fc21f54b8e..50d4e606fd 100755 --- a/scripts/exregional_run_enspointvx_prob.sh +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -28,7 +28,7 @@ set -x # #----------------------------------------------------------------------- # -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) # From 58ddaa4b14cca665115a61ac68f1c3495ef01b98 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 17 Nov 2021 15:32:21 -0500 Subject: [PATCH 101/203] Fix -np warning by RUN_CMD_FCST (#633) * move to easy fix --- ush/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.sh b/ush/setup.sh index 1dcc748abc..f00e3bf8dd 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -3064,7 +3064,7 @@ FVCOM_FILE="${FVCOM_FILE}" # NCORES_PER_NODE="${NCORES_PER_NODE}" PE_MEMBER01="${PE_MEMBER01}" -RUN_CMD_FCST=$(eval echo ${RUN_CMD_FCST}) +RUN_CMD_FCST="$(eval echo \'${RUN_CMD_FCST}\')" # #----------------------------------------------------------------------- # From 43fc3e8726825d003e31c164cfdda1cd7a6e7e9f Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 23 Nov 2021 13:45:42 -0500 Subject: [PATCH 102/203] Update template namelist files for newer ufs weather model (#642) * Update templates for new ufs weather model * Add option for pressure tendency diagnostic * Remove unnecessary namelist parameters * Modify model_configure * change nfhout to output_fh --- scripts/exregional_run_post.sh | 4 +- ush/create_model_configure_file.sh | 22 ++-- ush/templates/FV3.input.yml | 5 - ush/templates/input.nml.FV3 | 5 +- ush/templates/model_configure | 156 +++++++++-------------------- 5 files changed, 61 insertions(+), 131 deletions(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 2632d7d55e..025edd46ea 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -234,8 +234,8 @@ tmmark="tm00" # minutes and seconds of the corresponding output forecast time. # # Note that if the forecast model is instructed to output at some hourly -# interval (via the nfhout and nfhout_hf parameters in the MODEL_CONFIG_FN -# file, with nsout set to a non-positive value), then the write-component +# interval (via the output_fh parameter in the MODEL_CONFIG_FN file, +# with nsout set to a non-positive value), then the write-component # output file names will not contain any suffix for the minutes and seconds. # For this reason, when SUB_HOURLY_POST is not set to "TRUE", mnts_sec_str # must be set to a null string. diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index ccd55d00fa..a5d6c5d6f8 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -114,6 +114,7 @@ run directory (run_dir): # settings="\ 'PE_MEMBER01': ${PE_MEMBER01} + 'print_esmf': ${dot_print_esmf_dot} 'start_year': $yyyy 'start_month': $mm 'start_day': $dd @@ -125,7 +126,6 @@ run directory (run_dir): 'restart_interval': ${RESTART_INTERVAL} 'write_dopost': ${dot_write_dopost} 'quilting': ${dot_quilting_dot} - 'print_esmf': ${dot_print_esmf_dot} 'output_grid': ${WRTCMP_output_grid}" # 'output_grid': \'${WRTCMP_output_grid}\'" # @@ -185,17 +185,15 @@ run directory (run_dir): # main time step dt_atmos (in units of seconds). Note that nsout is # guaranteed to be an integer because the experiment generation scripts # require that dt_subhourly_post_mnts (after conversion to seconds) be -# evenly divisible by dt_atmos. Also, in this case, the variable nfhout -# [which specifies the (low-frequency) output interval in hours after -# forecast hour nfhmax_hf; see the jinja model_config template file] is -# set to 0, although this doesn't matter because any positive of nsout -# will override nfhout. +# evenly divisible by dt_atmos. Also, in this case, the variable output_fh +# [which specifies the output interval in hours; +# see the jinja model_config template file] is set to 0, although this +# doesn't matter because any positive of nsout will override output_fh. # # If sub_hourly_post is set to "FALSE", then the workflow is hard-coded # (in the jinja model_config template file) to direct the forecast model -# to output files every hour. This is done by setting (1) nfhout_hf to -# 1 in that jinja template file, (2) nfhout to 1 here, and (3) nsout to -# -1 here which turns off output by time step interval. +# to output files every hour. This is done by setting (1) output_fh to 1 +# here, and (2) nsout to -1 here which turns off output by time step interval. # # Note that the approach used here of separating how hourly and subhourly # output is handled should be changed/generalized/simplified such that @@ -208,13 +206,13 @@ run directory (run_dir): # if [ "${sub_hourly_post}" = "TRUE" ]; then nsout=$(( dt_subhourly_post_mnts*60 / dt_atmos )) - nfhout=0 + output_fh=0 else - nfhout=1 + output_fh=1 nsout=-1 fi settings="${settings} - 'nfhout': ${nfhout} + 'output_fh': ${output_fh} 'nsout': ${nsout}" print_info_msg $VERBOSE " diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 8e9b43fe5d..ce8f833f5b 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -288,11 +288,6 @@ FV3_GFS_v15p2: surf_map_nml: FV3_GFS_v16: - atmos_model_nml: - fhmax: 240 - fhmaxhf: 0 - fhout: 3 - fhouthf: 1 cires_ugwp_nml: launch_level: 27 fv_core_nml: diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index c4cec33295..2d35ca4770 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -10,7 +10,6 @@ &atmos_model_nml chksum_debug = .false. dycore_only = .false. - fdiag = 1 / &cires_ugwp_nml @@ -71,6 +70,7 @@ do_schmidt = .true. do_vort_damp = .true. dwind_2d = .false. + dz_min = 2 external_eta = .true. external_ic = .true. fill = .true. @@ -109,6 +109,7 @@ p_fac = 0.1 phys_hydrostatic = .false. print_freq = 6 + psm_bc = 1 range_warn = .true. read_increment = .false. regional = .true. @@ -174,7 +175,6 @@ ltaerosol = .true. lwhtr = .true. n_var_lndp = 0 - ncld = 5 nsradar_reset = 3600 nst_anl = .true. nstf_name = 2,1,0,0,0 @@ -182,6 +182,7 @@ oz_phys_2015 = .true. pdfcld = .false. pre_rad = .false. + print_diff_pgr = .false. prslrd0 = 0.0 random_clds = .false. redrag = .true. diff --git a/ush/templates/model_configure b/ush/templates/model_configure index f26b9dd15c..1b0f7a815a 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -1,5 +1,6 @@ total_member: 1 PE_MEMBER01: {{ PE_MEMBER01 }} +print_esmf: {{ print_esmf }} start_year: {{ start_year }} start_month: {{ start_month }} start_day: {{ start_day }} @@ -14,14 +15,12 @@ cpl: {{ cpl }} calendar: 'julian' memuse_verbose: .false. atmos_nthreads: {{ atmos_nthreads }} -use_hyper_thread: .false. -debug_affinity: .true. restart_interval: {{ restart_interval }} output_1st_tstep_rst: .false. -print_esmf: {{ print_esmf }} write_dopost: {{ write_dopost }} +ideflate: 0 +nbits: 0 quilting: {{ quilting }} - {% if quilting %} # # Write-component (quilting) computational parameters. @@ -29,29 +28,15 @@ quilting: {{ quilting }} write_groups: {{ write_groups }} write_tasks_per_group: {{ write_tasks_per_group }} num_files: 2 -filename_base: 'dyn''phy' -output_file: 'netcdf' -write_nemsioflip: .false. -write_fsyncflag: .false. +filename_base: 'dyn' 'phy' +output_file: 'netcdf' 'netcdf' # # Write-component output frequency parameter definitions: # -# nfhout: -# Output frequency in hours after forecast hour "nfhmax_hf". -# -# nfhmax_hf: -# Number of forecast hours until output frequency "nfhout" takes affect. +# output_fh: Output frequency in hours. +# nsout: Output frequency in time steps (positive values override "output_fh"). # -# nfhout_hf: -# Output frequency in hours until forecast hour "nfhmax_hf". -# -# nsout: -# Output frequency in time steps (positive values override "nfhout" and -# "nfhout_hf"). -# -nfhout: {{ nfhout }} -nfhmax_hf: 60 -nfhout_hf: 1 +output_fh: {{ output_fh }} -1 nsout: {{ nsout }} # # Coordinate system used by the output grid. @@ -61,97 +46,48 @@ output_grid: '{{ output_grid }}' # Parameter definitions for an output grid of type "{{ output_grid }}": # {%- if output_grid == "lambert_conformal" %} -# cen_lon: -# Longitude of center of grid (degrees). -# -# cen_lat: -# Latitude of center of grid (degrees). -# -# stdlat1: -# Latitude of first standard parallel (degrees). -# -# stdlat2: -# Latitude of second standard parallel (degrees). -# -# nx: -# Number of grid cells along x-axis in Lambert conformal (x,y) plane. -# -# ny: -# Number of grid cells along y-axis in Lambert conformal (x,y) plane. -# -# lon1: -# Longitude of center of grid cell at bottom-left corner of grid (degrees). -# -# lat1: -# Latitude of center of grid cell at bottom-left corner of grid (degrees). -# -# dx: -# Grid cell size in x direction (meters). -# -# dy: -# Grid cell size in y direction (meters). +# cen_lon: Longitude of center of grid (degrees). +# cen_lat: Latitude of center of grid (degrees). +# stdlat1: Latitude of first standard parallel (degrees). +# stdlat2: Latitude of second standard parallel (degrees). +# nx: Number of grid cells along x-axis in Lambert conformal (x,y) plane. +# ny: Number of grid cells along y-axis in Lambert conformal (x,y) plane. +# lon1: Longitude of center of grid cell at bottom-left corner of grid (degrees). +# lat1: Latitude of center of grid cell at bottom-left corner of grid (degrees). +# dx: Grid cell size in x direction (meters). +# dy: Grid cell size in y direction (meters). # {%- elif output_grid == "regional_latlon" %} -# cen_lon: -# Longitude of center of grid (degrees). -# -# cen_lat: -# Latitude of center of grid (degrees). -# -# lon1: -# Longitude of center of lower-left (southwest) grid cell (degrees). -# -# lat1: -# Latitude of center of lower-left (southwest) grid cell (degrees). -# -# lon2: -# Longitude of center of upper-right (northeast) grid cell (degrees). -# -# lat2: -# Latitude of center of upper-right (northeast) grid cell (degrees). -# -# dlon: -# Longitudinal grid size (degrees). -# -# dlat: -# Latitudinal grid size (degrees). +# cen_lon: Longitude of center of grid (degrees). +# cen_lat: Latitude of center of grid (degrees). +# lon1: Longitude of center of lower-left (southwest) grid cell (degrees). +# lat1: Latitude of center of lower-left (southwest) grid cell (degrees). +# lon2: Longitude of center of upper-right (northeast) grid cell (degrees). +# lat2: Latitude of center of upper-right (northeast) grid cell (degrees). +# dlon: Longitudinal grid size (degrees). +# dlat: Latitudinal grid size (degrees). # {%- elif output_grid == "rotated_latlon" %} -# cen_lon: -# Longitude of center of grid, expressed in the NON-ROTATED latlon -# coordinate system (degrees). This is also the longitude of the point -# at which the equator and prime meridian of the ROTATED coordinate -# system intersect (i.e. the point at which the longitude and latitude -# in the ROTATED latlon coordinate system are both 0). -# -# cen_lat: -# Latitude of center of grid, expressed in the NON-ROTATED latlon -# coordinate system (degrees). This is also the latitude of the point -# at which the equator and prime meridian of the ROTATED coordinate system -# intersect (i.e. the point at which the longitude and latitude in the -# ROTATED latlon coordinate system are both 0). -# -# lon1: -# Longitude of center of lower-left grid cell, expressed in the ROTATED -# latlon coordinate system (degrees). -# -# lat1: -# Latitude of center of lower-left grid cell, expressed in the ROTATED -# latlon coordinate system (degrees). -# -# lon2: -# Longitude of center of upper-right grid cell, expressed in the ROTATED -# latlon coordinate system (degrees). -# -# lat2: -# Latitude of center of upper-right grid cell, expressed in the ROTATED -# latlon coordinate system (degrees). -# -# dlon: -# Longitudinal grid size in the ROTATED latlon coordinate system (degrees). -# -# dlat: -# Latitudinal grid size in the ROTATED latlon coordinate system (degrees). +# cen_lon: Longitude of center of grid, expressed in the NON-ROTATED latlon coordinate +# system (degrees). This is also the longitude of the point at which the +# equator and prime meridian of the ROTATED coordinate system intersect (i.e. +# the point at which the longitude and latitude in the ROTATED latlon +# coordinate system are both 0). +# cen_lat: Latitude of center of grid, expressed in the NON-ROTATED latlon coordinate +# system (degrees). This is also the latitude of the point at which the +# equator and prime meridian of the ROTATED coordinate system intersect (i.e. +# the point at which the longitude and latitude in the ROTATED latlon +# coordinate system are both 0). +# lon1: Longitude of center of lower-left grid cell, expressed in the ROTATED latlon +# coordinate system (degrees). +# lat1: Latitude of center of lower-left grid cell, expressed in the ROTATED latlon +# coordinate system (degrees). +# lon2: Longitude of center of upper-right grid cell, expressed in the ROTATED latlon +# coordinate system (degrees). +# lat2: Latitude of center of upper-right grid cell, expressed in the ROTATED latlon +# coordinate system (degrees). +# dlon: Longitudinal grid size in the ROTATED latlon coordinate system (degrees). +# dlat: Latitudinal grid size in the ROTATED latlon coordinate system (degrees). # {%- endif %} {%- if output_grid == "lambert_conformal" %} From 755a2318536392726d1031dc48dbc50fbd528410 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 6 Dec 2021 10:34:05 -0700 Subject: [PATCH 103/203] Add new DEBUG variable (#639) ## DESCRIPTION OF CHANGES: 1. Add a new experiment configuration variable named `DEBUG` to enable more in-depth debugging output from workflow scripts. Set default value of `DEBUG` in `config_defaults.sh` to `"FALSE"`. 2. In experiment generation scripts, change circumstances under which different messages are printed to screen (e.g. when `VERBOSE` is `"TRUE"`, when `DEBUG` is `"TRUE"`, or always). 3. In experiment generation scripts, for clarity add new informational messages and modify some existing ones. 4. In various scripts, change "set -x" to "set +x" to reduce output clutter. This can be changed back as necessary (e.g. for debugging). Note that if `DEBUG` is set to `"TRUE"`, `VERBOSE` will get reset to `"TRUE"` if necessary in order to also print out all the `VERBOSE` messages. ## TESTS CONDUCTED: Ran the WE2E test `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` as-is as well as with modifications to the default values of `VERBOSE` and `DEBUG`, as follows: 1. `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` as-is, i.e. using default values `VERBOSE="TRUE"` and `DEBUG="FALSE"`. 2. `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` modified with `VERBOSE="FALSE"` (and with default of `DEBUG="FALSE"`). 3. `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` modified with `DEBUG="TRUE"` (and with default of `VERBOSE="TRUE"`). 4. `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` modified with `DEBUG="TRUE"` and `VERBOSE="FALSE"` (which should get reset to `"TRUE"`). All tests were successful. The experiment generation log files (`log.generate_FV3LAM_wflow.sh`) were compared and differed in the expected ways. ## DOCUMENTATION: Necessary documentation of `DEBUG` is in `config_defaults.sh`. Created Issue #[640 ](https://github.com/NOAA-EMC/regional_workflow/issues/640)to also update rst documentation. --- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_MAKE_ICS | 2 +- jobs/JREGIONAL_RUN_POST | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- ush/bash_utils/print_input_args.sh | 9 ++- ush/bash_utils/set_file_param.sh | 22 +++---- ush/check_ruc_lsm.sh | 2 +- ush/config_defaults.sh | 14 +++- ush/create_diag_table_file.sh | 2 +- ush/create_model_configure_file.sh | 2 +- ush/generate_FV3LAM_wflow.sh | 95 ++++++++++++++++----------- ush/link_fix.sh | 2 +- ush/make_grid_mosaic_file.sh | 2 +- ush/set_FV3nml_sfc_climo_filenames.sh | 2 +- ush/set_FV3nml_stoch_params.sh | 2 +- ush/set_ozone_param.sh | 4 +- ush/set_thompson_mp_fix_files.sh | 2 +- ush/setup.sh | 70 ++++++++++++++++---- ush/valid_param_vals.sh | 1 + 28 files changed, 164 insertions(+), 93 deletions(-) diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index e63010d0d6..bd1760ee0d 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -29,7 +29,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index dfe1f68a6c..f5d6349745 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 9514931cc1..ab44e21b8f 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 9c5125f418..a5e5894f10 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index ed469dbe28..86a30470bb 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index 641df6f663..867c75fc84 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index d8c4dcea04..a75558d928 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index 7ae543314b..a8a2c43be5 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index c240d7abeb..fc30f076fb 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index 376d07b99f..a7d937c12e 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 1a47cfd124..818e588e51 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index b17fd29605..8537cb3ec5 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index f995045aef..084543695f 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 72b7744b38..957cec5243 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -168,16 +168,15 @@ have been set as follows: # #----------------------------------------------------------------------- # -# If a global variable named VERBOSE is not defined, print out the mes- -# sage. If it is defined, print out the message only if VERBOSE is set -# to TRUE. +# If a global variable named DEBUG is not defined, print out the message. +# If it is defined, print out the message only if DEBUG is set to TRUE. # #----------------------------------------------------------------------- # - if [ -z ${VERBOSE+x} ]; then + if [ -z ${DEBUG+x} ]; then print_info_msg "$msg" else - print_info_msg "$VERBOSE" "$msg" + print_info_msg "$DEBUG" "$msg" fi # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/set_file_param.sh b/ush/bash_utils/set_file_param.sh index d454514831..3e0d133494 100644 --- a/ush/bash_utils/set_file_param.sh +++ b/ush/bash_utils/set_file_param.sh @@ -53,11 +53,11 @@ Incorrect number of arguments specified: Usage: - ${func_name} file_full_path param value + ${func_name} file_fp param value where the arguments are defined as follows: - file_full_path: + file_fp: Full path to the file in which the specified parameter's value will be set. @@ -76,7 +76,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local file_full_path="$1" + local file_fp="$1" local param="$2" local value="$3" # @@ -86,7 +86,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local file="${file_full_path##*/}" + local file="${file_fp##*/}" # #----------------------------------------------------------------------- # @@ -94,7 +94,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - print_info_msg "$VERBOSE" "\ + print_info_msg "$DEBUG" "\ Setting parameter \"$param\" in file \"$file\" to \"$value\" ..." # #----------------------------------------------------------------------- @@ -141,7 +141,7 @@ Setting parameter \"$param\" in file \"$file\" to \"$value\" ..." "${GLOBAL_VAR_DEFNS_FN}") regex_search="(^\s*$param=)(\".*\")?([^ \"]*)?(\(.*\))?(\s*[#].*)?" regex_replace="\1$value\5" -# set_bash_param "${file_full_path}" "$param" "$value" +# set_bash_param "${file_fp}" "$param" "$value" ;; # #----------------------------------------------------------------------- @@ -168,15 +168,15 @@ specified for this file: # #----------------------------------------------------------------------- # - grep -q -E "${regex_search}" "${file_full_path}" + grep -q -E "${regex_search}" "${file_fp}" if [ $? -eq 0 ]; then - $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" + $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_fp}" else print_err_msg_exit "\ -Specified file (file_full_path) does not contain the searched-for regu- -lar expression (regex_search): - file_full_path = \"${file_full_path}\" +The specified file (file_fp) does not contain the searched-for regular +expression (regex_search): + file_fp = \"${file_fp}\" param = \"$param\" value = \"$value\" regex_search = ${regex_search}" diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh index 8d51cc01ba..35e4db195f 100644 --- a/ush/check_ruc_lsm.sh +++ b/ush/check_ruc_lsm.sh @@ -20,7 +20,7 @@ function check_ruc_lsm() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index ba3711f906..4bdfdfc6ed 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1171,13 +1171,21 @@ PREEXISTING_DIR_METHOD="delete" # #----------------------------------------------------------------------- # -# Set VERBOSE. This is a flag that determines whether or not the experiment -# generation and workflow task scripts tend to print out more informational -# messages. +# Set flags for more detailed messages. Defintitions: +# +# VERBOSE: +# This is a flag that determines whether or not the experiment generation +# and workflow task scripts tend to print out more informational messages. +# +# DEBUG: +# This is a flag that determines whether or not very detailed debugging +# messages are printed to out. Note that if DEBUG is set to TRUE, then +# VERBOSE will also get reset to TRUE if it isn't already. # #----------------------------------------------------------------------- # VERBOSE="TRUE" +DEBUG="FALSE" # #----------------------------------------------------------------------- # diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 3c409031a6..41ca941bf8 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -15,7 +15,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index a5d6c5d6f8..8141877650 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -15,7 +15,7 @@ function create_model_configure_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 0ee8eee0b8..a9fc3a8171 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -10,6 +10,15 @@ #----------------------------------------------------------------------- # function generate_FV3LAM_wflow() { +printf "\ +======================================================================== +======================================================================== + +Starting experiment generation... + +======================================================================== +======================================================================== +" # #----------------------------------------------------------------------- # @@ -20,10 +29,10 @@ function generate_FV3LAM_wflow() { #----------------------------------------------------------------------- # if [[ $(uname -s) == Darwin ]]; then - local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) - else - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) - fi + local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) +else + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +fi local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -108,7 +117,6 @@ if [ $pyerrors -gt 0 ];then " fi - # #----------------------------------------------------------------------- # @@ -152,16 +160,26 @@ WFLOW_XML_FP="$EXPTDIR/${WFLOW_XML_FN}" # #----------------------------------------------------------------------- # -ensmem_indx_name="\"\"" -uscore_ensmem_name="\"\"" -slash_ensmem_subdir="\"\"" -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - ensmem_indx_name="mem" - uscore_ensmem_name="_mem#${ensmem_indx_name}#" - slash_ensmem_subdir="/mem#${ensmem_indx_name}#" -fi +if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then -settings="\ + template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" + + print_info_msg " +Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML +file (template_xml_fp): + template_xml_fp = \"${template_xml_fp}\" + WFLOW_XML_FP = \"${WFLOW_XML_FP}\"" + + ensmem_indx_name="\"\"" + uscore_ensmem_name="\"\"" + slash_ensmem_subdir="\"\"" + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + ensmem_indx_name="mem" + uscore_ensmem_name="_mem#${ensmem_indx_name}#" + slash_ensmem_subdir="/mem#${ensmem_indx_name}#" + fi + + settings="\ # # Parameters needed by the job scheduler. # @@ -415,7 +433,7 @@ settings="\ 'first_fv3_file_tstr': "000:"`$DATE_UTIL -d "${DATE_FIRST_CYCL} +${DT_ATMOS} seconds" +%M:%S` " # End of "settings" variable. -print_info_msg $VERBOSE " + print_info_msg "$VERBOSE" " The variable \"settings\" specifying values of the rococo XML variables has been set as follows: #----------------------------------------------------------------------- @@ -423,12 +441,9 @@ settings = $settings" # -# Set the full path to the template rocoto XML file. Then call a python -# script to generate the experiment's actual XML file from this template -# file. +# Call the python script to generate the experiment's actual XML file +# from the jinja template file. # -if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then - template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" $USHDIR/fill_jinja_template.py -q \ -u "${settings}" \ -t ${template_xml_fp} \ @@ -444,6 +459,7 @@ are: Namelist settings specified on command line: settings = $settings" + fi # #----------------------------------------------------------------------- @@ -453,7 +469,7 @@ fi # #----------------------------------------------------------------------- # -print_info_msg " +print_info_msg "$VERBOSE" " Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): EXPTDIR = \"${EXPTDIR}\" @@ -476,7 +492,7 @@ if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then # time_stamp=$( $DATE_UTIL "+%F_%T" ) crontab_backup_fp="$EXPTDIR/crontab.bak.${time_stamp}" - print_info_msg " + print_info_msg "$VERBOSE" " Copying contents of user cron table to backup file: crontab_backup_fp = \"${crontab_backup_fp}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then @@ -522,9 +538,9 @@ added: else - print_info_msg " -Adding the following line to the cron table in order to automatically -resubmit FV3-LAM workflow: + print_info_msg "$VERBOSE" " +Adding the following line to the user's cron table in order to automatically +resubmit SRW workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ];then @@ -637,8 +653,8 @@ cp_vrfy "${FIELD_DICT_IN_UWM_FP}" "${FIELD_DICT_FP}" # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " -Setting parameters in FV3 namelist file (FV3_NML_FP): +print_info_msg " +Setting parameters in weather model's namelist file (FV3_NML_FP): FV3_NML_FP = \"${FV3_NML_FP}\"" # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. @@ -800,8 +816,8 @@ settings="$settings }" print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the namelist variables -has been set as follows: +The variable \"settings\" specifying values of the weather model's +namelist variables has been set as follows: settings = $settings" @@ -886,15 +902,12 @@ print_info_msg " ======================================================================== ======================================================================== -Workflow generation completed. +Experiment generation completed. The experiment directory is: + + EXPTDIR=\"$EXPTDIR\" ======================================================================== ======================================================================== - -The experiment directory is: - - > EXPTDIR=\"$EXPTDIR\" - " # #----------------------------------------------------------------------- @@ -904,12 +917,16 @@ The experiment directory is: #----------------------------------------------------------------------- # if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then + print_info_msg "\ To launch the workflow, first ensure that you have a compatible version of rocoto available. For most pre-configured platforms, rocoto can be loaded via a module: + > module load rocoto + For more details on rocoto, see the User's Guide. + To launch the workflow, first ensure that you have a compatible version of rocoto loaded. For example, to load version 1.3.1 of rocoto, use @@ -938,17 +955,15 @@ Note that: 2) In order for the output of the rocotostat command to be up-to-date, the rocotorun command must be issued immediately before the rocoto- stat command. -" -fi -print_info_msg " + For automatic resubmission of the workflow (say every 3 minutes), the following line can be added to the user's crontab (use \"crontab -e\" to edit the cron table): */3 * * * * cd $EXPTDIR && ./launch_FV3LAM_wflow.sh - -Done. " + +fi # # If necessary, run the NOMADS script to source external model data. # diff --git a/ush/link_fix.sh b/ush/link_fix.sh index 34bcd5a53a..0bfd5c03d8 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -16,7 +16,7 @@ function link_fix() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 8a81feb146..5adf2ac30b 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh index 13f2ecdf5d..d6f9dc5d4c 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ b/ush/set_FV3nml_sfc_climo_filenames.sh @@ -20,7 +20,7 @@ function set_FV3nml_sfc_climo_filenames() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index cfe115d525..8f08bb566a 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -22,7 +22,7 @@ function set_FV3nml_stoch_params() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index c857492798..bd6f5fd377 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -34,7 +34,7 @@ function set_ozone_param() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -209,7 +209,7 @@ files in the FIXam directory is: " msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) msg="$msg"$( printf "\n)" ) - print_info_msg "$msg" + print_info_msg "$VERBOSE" "$msg" else diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 667ad522fa..7bad26c2bb 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -22,7 +22,7 @@ function set_thompson_mp_fix_files() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index f00e3bf8dd..83843eea80 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -51,6 +51,11 @@ cd_vrfy ${scrfunc_dir} #----------------------------------------------------------------------- # . ./source_util_funcs.sh + +print_info_msg " +======================================================================== +Starting function ${func_name}() in \"${scrfunc_fn}\"... +========================================================================" # #----------------------------------------------------------------------- # @@ -152,6 +157,40 @@ fi # #----------------------------------------------------------------------- # +# Make sure that DEBUG is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DEBUG" "valid_vals_DEBUG" +# +# Set DEBUG to either "TRUE" or "FALSE" so we don't have to consider +# other valid values later on. +# +DEBUG=$(echo_uppercase $DEBUG) +if [ "$DEBUG" = "TRUE" ] || \ + [ "$DEBUG" = "YES" ]; then + DEBUG="TRUE" +elif [ "$DEBUG" = "FALSE" ] || \ + [ "$DEBUG" = "NO" ]; then + DEBUG="FALSE" +fi +# +#----------------------------------------------------------------------- +# +# If DEBUG is set to "TRUE" but VERBOSE is set to "FALSE", reset VERBOSE +# to "TRUE" to print out all of the VERBOSE output (in addition to any +# DEBUG output). +# +#----------------------------------------------------------------------- +# +if [ "$DEBUG" = "TRUE" ] && [ "$VERBOSE" = "FALSE" ]; then + print_info_msg " +Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." + VERBOSE="TRUE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that USE_CRON_TO_RELAUNCH is set to a valid value. # #----------------------------------------------------------------------- @@ -792,9 +831,9 @@ fi # #----------------------------------------------------------------------- # -DATE_OR_NULL=$( printf "%s" "${DATE_FIRST_CYCL}" | \ +date_or_null=$( printf "%s" "${DATE_FIRST_CYCL}" | \ $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${DATE_OR_NULL}" ]; then +if [ -z "${date_or_null}" ]; then print_err_msg_exit "\ DATE_FIRST_CYCL must be a string consisting of exactly 8 digits of the form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit @@ -802,9 +841,9 @@ month, and DD is the 2-digit day-of-month. DATE_FIRST_CYCL = \"${DATE_FIRST_CYCL}\"" fi -DATE_OR_NULL=$( printf "%s" "${DATE_LAST_CYCL}" | \ +date_or_null=$( printf "%s" "${DATE_LAST_CYCL}" | \ $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${DATE_OR_NULL}" ]; then +if [ -z "${date_or_null}" ]; then print_err_msg_exit "\ DATE_LAST_CYCL must be a string consisting of exactly 8 digits of the form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit @@ -2549,8 +2588,9 @@ line_list=$( $SED -r \ -e "/^$/d" \ ${GLOBAL_VAR_DEFNS_FP} ) -print_info_msg "$VERBOSE" " -The variable \"line_list\" contains: +print_info_msg "$DEBUG" " +Before updating default values of experiment variables to user-specified +values, the variable \"line_list\" contains: ${line_list} " @@ -2593,6 +2633,17 @@ $SED -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} # # Loop through the lines in line_list. # +print_info_msg " +Generating the global experiment variable definitions file specified by +GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" +Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" +For more detailed information, set DEBUG to \"TRUE\" in the experiment +configuration file (\"${EXPT_CONFIG_FN}\")." + +template_var_names=() +template_var_values=() while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. @@ -2604,16 +2655,13 @@ while read crnt_line; do # set to. # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) -#echo -#echo "============================" -#printf "%s\n" "var_name = \"${var_name}\"" # # If var_name is not empty, then a variable name was found in the cur- # rent line in line_list. # if [ ! -z $var_name ]; then - print_info_msg "$VERBOSE" " + print_info_msg "$DEBUG" " var_name = \"${var_name}\"" # # If the variable specified in var_name is set in the current environ- @@ -3087,7 +3135,7 @@ definitions file returned with a nonzero status." # print_info_msg " ======================================================================== -Setup script completed successfully!!! +Function ${func_name}() in \"${scrfunc_fn}\" completed successfully!!! ========================================================================" # #----------------------------------------------------------------------- diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 1b3acf00ae..8104038da9 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -3,6 +3,7 @@ # valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") From 83b4d3873c3ef05bbf9d55e90da61e4807cea68c Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 6 Dec 2021 14:57:59 -0700 Subject: [PATCH 104/203] Toggle on/off workflow tasks (#645) ## DESCRIPTION OF CHANGES: 1. Add capability to turn on/off those workflow tasks that currently do not have this capability. These tasks are: a. Getting the external model files for creating initial conditions (`GET_EXTRN_ICS_TN`). b. Getting the external model files for creating boundary conditions (`GET_EXTRN_LBCS_TN`). c. Regridding/interpolating data from the external model IC files to obtain IC fields on the native grid (`MAKE_ICS_TN`). d. Regridding/interpolating data from the external model LBC files to obtain LBC fields on the native grid (`MAKE_LBCS_TN`). e. Running the forecast (`RUN_FCST`). Note that any remaining tasks in the workflow already have the capability to be toggled on/off. 2. Add a WE2E test (named `deactivate_tasks`) to test the ability to turn off tasks. In this test, all the tasks except `MAKE_GRID_TN`, `MAKE_OROG_TN`, and `MAKE_SFC_CLIMO_TN` are turned off (the latter 3 are not turned off because there is already a separate WE2E for turning those three off). ## TESTS CONDUCTED: Ran the new WE2E test (`deactivate_tasks`) successfully on Hera. ## DOCUMENTATION: Necessary documentation has been included in `config_defaults.sh`. Created Issue #646 to also update rst documentation. --- .../wflow_features/config.deactivate_tasks.sh | 26 ++++++ ush/config_defaults.sh | 26 ++++-- ush/generate_FV3LAM_wflow.sh | 5 ++ ush/setup.sh | 63 +++++++------ ush/templates/FV3LAM_wflow.xml | 88 +++++++++++-------- 5 files changed, 131 insertions(+), 77 deletions(-) create mode 100644 tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh diff --git a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh new file mode 100644 index 0000000000..52aad5cfa6 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test ensures that the various workflow tasks can be deactivated, +# i.e. removed from the Rocoto XML. Note that we leave the MAKE_GRID_TN, +# MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because there is a +# separate test for turning those off. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +RUN_TASK_GET_EXTRN_ICS="FALSE" +RUN_TASK_GET_EXTRN_LBCS="FALSE" +RUN_TASK_MAKE_ICS="FALSE" +RUN_TASK_MAKE_LBCS="FALSE" +RUN_TASK_RUN_FCST="FALSE" +RUN_TASK_RUN_POST="FALSE" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 4bdfdfc6ed..8c2ba74aae 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1265,6 +1265,21 @@ VX_ENSPOINT_PROB_TN="run_enspointvx_prob" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # +# RUN_TASK_GET_EXTRN_ICS: +# Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. +# +# RUN_TASK_GET_EXTRN_LBCS: +# Flag that determines whether the GET_EXTRN_LBCS_TN task is to be run. +# +# RUN_TASK_MAKE_ICS: +# Flag that determines whether the MAKE_ICS_TN task is to be run. +# +# RUN_TASK_MAKE_LBCS: +# Flag that determines whether the MAKE_LBCS_TN task is to be run. +# +# RUN_TASK_RUN_FCST: +# Flag that determines whether the RUN_FCST_TN task is to be run. +# # RUN_TASK_RUN_POST: # Flag that determines whether the RUN_POST_TN task is to be run. # @@ -1296,20 +1311,19 @@ OROG_DIR="/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO="TRUE" SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files" +RUN_TASK_GET_EXTRN_ICS="TRUE" +RUN_TASK_GET_EXTRN_LBCS="TRUE" +RUN_TASK_MAKE_ICS="TRUE" +RUN_TASK_MAKE_LBCS="TRUE" +RUN_TASK_RUN_FCST="TRUE" RUN_TASK_RUN_POST="TRUE" RUN_TASK_GET_OBS_CCPA="FALSE" - RUN_TASK_GET_OBS_MRMS="FALSE" - RUN_TASK_GET_OBS_NDAS="FALSE" - RUN_TASK_VX_GRIDSTAT="FALSE" - RUN_TASK_VX_POINTSTAT="FALSE" - RUN_TASK_VX_ENSGRID="FALSE" - RUN_TASK_VX_ENSPOINT="FALSE" # #----------------------------------------------------------------------- diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index a9fc3a8171..1370a7bc4d 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -361,6 +361,11 @@ file (template_xml_fp): 'run_task_make_grid': ${RUN_TASK_MAKE_GRID} 'run_task_make_orog': ${RUN_TASK_MAKE_OROG} 'run_task_make_sfc_climo': ${RUN_TASK_MAKE_SFC_CLIMO} + 'run_task_get_extrn_ics': ${RUN_TASK_GET_EXTRN_ICS} + 'run_task_get_extrn_lbcs': ${RUN_TASK_GET_EXTRN_LBCS} + 'run_task_make_ics': ${RUN_TASK_MAKE_ICS} + 'run_task_make_lbcs': ${RUN_TASK_MAKE_LBCS} + 'run_task_run_fcst': ${RUN_TASK_RUN_FCST} 'run_task_run_post': ${RUN_TASK_RUN_POST} 'run_task_get_obs_ccpa': ${RUN_TASK_GET_OBS_CCPA} 'run_task_get_obs_mrms': ${RUN_TASK_GET_OBS_MRMS} diff --git a/ush/setup.sh b/ush/setup.sh index 83843eea80..9cc9971328 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2608,10 +2608,10 @@ read -r -d '' str_to_insert << EOM #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 1: -# This section is a copy of the default workflow/experiment configura- -# tion file config_defaults.sh in the shell scripts directory USHDIR ex- -# cept that variable values have been updated to those set by the setup -# script (setup.sh). +# This section is a copy of the default experiment configuration file +# (${EXPT_DEFAULT_CONFIG_FN}) in the shell scripts directory specified by USHDIR +# except that variable values have been updated to those for the experiment +# (as opposed to the default values). #----------------------------------------------------------------------- #----------------------------------------------------------------------- # @@ -2620,7 +2620,7 @@ EOM # Replace all occurrences of actual newlines in the variable str_to_insert # with escaped backslash-n. This is needed for the sed command below to # work properly (i.e. to avoid it failing with an "unterminated `s' command" -# message). +# error message). # str_to_insert=${str_to_insert//$'\n'/\\n} # @@ -2649,25 +2649,24 @@ while read crnt_line; do # Try to obtain the name of the variable being set on the current line. # This will be successful only if the line consists of one or more char- # acters representing the name of a variable (recall that in generating -# the variable line_list, all leading spaces in the lines in the file -# have been stripped out), followed by an equal sign, followed by zero -# or more characters representing the value that the variable is being -# set to. +# the variable line_list, leading spaces on each line were stripped out), +# followed by an equal sign, followed by zero or more characters +# representing the value that the variable is being set to. # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) # -# If var_name is not empty, then a variable name was found in the cur- -# rent line in line_list. +# If var_name is not empty, then a variable name was found on the current +# line in line_list. # if [ ! -z $var_name ]; then print_info_msg "$DEBUG" " var_name = \"${var_name}\"" # -# If the variable specified in var_name is set in the current environ- -# ment (to either an empty or non-empty string), get its value and in- -# sert it in the variable definitions file on the line where that varia- -# ble is defined. Note that +# If the variable specified in var_name is set in the current environment +# (to either an empty or non-empty string), get its value and insert it +# in the variable definitions file on the line where that variable is +# defined. Note that # # ${!var_name+x} # @@ -2687,13 +2686,13 @@ var_name = \"${var_name}\"" # # We will now set the variable var_value to the string that needs to be # placed on the right-hand side of the assignment operator (=) on the -# appropriate line in variable definitions file. How this is done de- -# pends on whether the variable is a scalar or an array. +# appropriate line in the variable definitions file. How this is done +# depends on whether the variable is a scalar or an array. # # If the variable contains only one element, then it is a scalar. (It -# could be a 1-element array, but it is simpler to treat it as a sca- -# lar.) In this case, we enclose its value in double quotes and save -# the result in var_value. +# could be a 1-element array, but for simplicity, we treat that case as +# a scalar.) In this case, we enclose its value in double quotes and +# save the result in var_value. # if [ "$num_elems" -eq 1 ]; then var_value="${!var_name}" @@ -2707,8 +2706,8 @@ var_name = \"${var_name}\"" # # 2) Place parentheses around the double-quoted list of array elements # generated in the first step. Note that there is no need to put a -# space before the closing parenthesis because in step 1, we have al- -# ready placed a space after the last element. +# space before the closing parenthesis because in step 1, we have +# already placed a space after the last element. # else @@ -2731,16 +2730,16 @@ var_name = \"${var_name}\"" fi # -# If the variable specified in var_name is not set in the current envi- -# ronment (to either an empty or non-empty string), get its value and -# insert it in the variable definitions file on the line where that va- -# riable is defined. +# If for some reason the variable specified in var_name is not set in +# the current environment (to either an empty or non-empty string), below +# we will still include it in the variable definitions file and simply +# set it to a null string. Thus, here, we set its value (var_value) to +# an empty string). In this case, we also issue an informational message. # else print_info_msg " -The variable specified by \"var_name\" is not set in the current envi- -ronment: +The variable specified by \"var_name\" is not set in the current environment: var_name = \"${var_name}\" Setting its value in the variable definitions file to an empty string." @@ -2749,13 +2748,13 @@ Setting its value in the variable definitions file to an empty string." fi # # Now place var_value on the right-hand side of the assignment statement -# on the appropriate line in variable definitions file. +# on the appropriate line in the variable definitions file. # set_file_param "${GLOBAL_VAR_DEFNS_FP}" "${var_name}" "${var_value}" # -# If var_name is empty, then a variable name was not found in the cur- -# rent line in line_list. In this case, print out a warning and move on -# to the next line. +# If var_name is empty, then a variable name was not found on the current +# line in line_list. In this case, print out a warning and move on to +# the next line. # else diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index cc77b5ad40..e136ff8d85 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,45 +22,45 @@ Parameters needed by the job scheduler. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + &RSRV_HPSS; {%- if machine in ["WCOSS_CRAY"] %} @@ -291,7 +291,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; {%- if machine in ["WCOSS_CRAY"] %} @@ -331,7 +331,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {%- if run_task_make_ics %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_ICS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_ICS" @@ -375,7 +375,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_LBCS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_LBCS" @@ -419,7 +419,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_FCST; &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" @@ -474,7 +474,7 @@ later below for other output times. 000 00 - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" @@ -528,7 +528,7 @@ for other output times. {% for min in range(delta_min, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" @@ -573,7 +573,7 @@ output hours (and all output minutes for each such hour). {% for h in range(1, fcst_len_hrs) %}{{ " %03d" % h }}{% endfor %} {% for min in range(0, 60, delta_min) %}{{ " %02d" % min }}{% endfor %} - + {%- else %} {% for h in range(0, fcst_len_hrs+1) %}{{ " %03d" % h }}{% endfor %} - + {%- endif %} &RSRV_DEFAULT; @@ -655,7 +655,7 @@ the tag to be identical to the ones above for other output times. {{ "%03d" % fcst_len_hrs }} 00 - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" @@ -695,7 +695,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; {%- if machine in ["WCOSS_CRAY"] %} @@ -728,7 +728,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; {%- if machine in ["WCOSS_CRAY"] %} @@ -762,7 +762,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_HPSS; {%- if machine in ["WCOSS_CRAY"] %} @@ -794,7 +794,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -844,7 +844,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -893,7 +893,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -942,7 +942,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -977,7 +977,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -1012,7 +1012,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_GRIDSTAT" @@ -1047,7 +1047,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_POINTSTAT" @@ -1099,7 +1099,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1131,7 +1131,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1163,7 +1163,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1195,7 +1195,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1226,7 +1226,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1256,7 +1256,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID" @@ -1285,7 +1285,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" @@ -1316,7 +1316,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1347,7 +1347,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" @@ -1378,7 +1378,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1410,7 +1410,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" @@ -1441,7 +1441,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1473,7 +1473,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_MEAN" @@ -1504,7 +1504,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1535,7 +1535,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1565,7 +1565,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSGRID_PROB" @@ -1596,7 +1596,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT" @@ -1624,7 +1624,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT_MEAN" @@ -1652,7 +1652,7 @@ the tag to be identical to the ones above for other output times. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&VX_TN;" "&JOBSDIR;/JREGIONAL_RUN_VX_ENSPOINT_PROB" From b6bbca27cd82105d2d54fc4b60c53305e08ad708 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 13 Dec 2021 14:36:40 -0500 Subject: [PATCH 107/203] Make thompson_mynn_lam3km ccpp suite available (#644) * Add FV3_GFS_v15_thompson_mynn_lam3km * Add a we2e test for thompson_mynn_lam3km * Update namelist files * Add MERRA2 data * Fix typos * Move the SDF line in scripts * remove change of nstf_name --- scripts/exregional_make_ics.sh | 1 + scripts/exregional_make_lbcs.sh | 1 + scripts/exregional_run_fcst.sh | 22 ++ ...3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh | 25 ++ ush/config_defaults.sh | 8 + ush/generate_FV3LAM_wflow.sh | 22 +- ush/setup.sh | 28 +- ush/templates/FV3.input.yml | 48 ++- ...iag_table.FV3_GFS_v15_thompson_mynn_lam3km | 340 ++++++++++++++++++ ...eld_table.FV3_GFS_v15_thompson_mynn_lam3km | 51 +++ ush/templates/input.nml.FV3 | 2 + ush/valid_param_vals.sh | 1 + 12 files changed, 544 insertions(+), 5 deletions(-) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh create mode 100644 ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km create mode 100644 ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index c82a987add..67915c6095 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -187,6 +187,7 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ + "FV3_GFS_v15_thompson_mynn_lam3km" | \ "FV3_HRRR" ) if [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ]; then diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 01aeaed965..c28fe24ebb 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -186,6 +186,7 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ + "FV3_GFS_v15_thompson_mynn_lam3km" | \ "FV3_HRRR" ) if [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ]; then diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5a91b26c33..8aa05ce34d 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -405,6 +405,28 @@ done # #----------------------------------------------------------------------- # +# Create links in the current run directory to the MERRA2 aerosol +# climatology data files and lookup table for optics properties. +# +#----------------------------------------------------------------------- +# +for f_nm_path in ${FIXclim}/*; do + f_nm=$( basename "${f_nm_path}" ) + pre_f="${f_nm%%.*}" + + if [ "${pre_f}" = "merra2" ]; then + mnth=$( printf "%s\n" "${f_nm}" | grep -o -P '(?<=2014.m).*(?=.nc)' ) + symlink="${run_dir}/aeroclim.m${mnth}.nc" + else + symlink="${run_dir}/${pre_f}.dat" + fi + target="${f_nm_path}" + create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" +done +# +#----------------------------------------------------------------------- +# # If running this cycle/ensemble member combination more than once (e.g. # using rocotoboot), remove any time stamp file that may exist from the # previous attempt. diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh new file mode 100644 index 0000000000..f391b26886 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the GFS_v15_ +# thompson_mynn_lam3km physics suite with ICs and LBCs derived from FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +CCPP_PHYS_SUITE="FV3_GFS_v15_thompson_mynn_lam3km" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 56b54ad688..702cf4db8d 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1357,6 +1357,12 @@ SFC_CLIMO_FIELDS=( \ # System directory in which the majority of fixed (i.e. time-independent) # files that are needed to run the FV3-LAM model are located # +# FIXaer: +# System directory where MERRA2 aerosol climatology files are located +# +# FIXlut: +# System directory where the lookup tables for optics properties are located +# # TOPO_DIR: # The location on disk of the static input files used by the make_orog # task (orog.x and shave.x). Can be the same as FIXgsm. @@ -1416,6 +1422,8 @@ SFC_CLIMO_FIELDS=( \ # to a null string which will then be overwritten in setup.sh unless the # user has specified a different value in config.sh FIXgsm="" +FIXaer="" +FIXlut="" TOPO_DIR="" SFC_CLIMO_INPUT_DIR="" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 4d9054bbcb..d19a58b52c 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -614,6 +614,25 @@ fi # #----------------------------------------------------------------------- # +# Copy MERRA2 aerosol climatology data. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Copying MERRA2 aerosol climatology data files from system directory +(FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: + FIXaer = \"${FIXaer}\" + FIXlut = \"${FIXlut}\" + FIXclim = \"${FIXclim}\"" + +check_for_preexist_dir_file "${FIXclim}" "delete" +mkdir_vrfy -p "${FIXclim}" + +cp_vrfy "${FIXaer}/merra2.aerclim"*".nc" "${FIXclim}/" +cp_vrfy "${FIXlut}/optics"*".dat" "${FIXclim}/" +# +#----------------------------------------------------------------------- +# # Copy templates of various input files to the experiment directory. # #----------------------------------------------------------------------- @@ -630,8 +649,7 @@ print_info_msg "$VERBOSE" " cp_vrfy "${FIELD_TABLE_TMPL_FP}" "${FIELD_TABLE_FP}" print_info_msg "$VERBOSE" " - Copying the template NEMS configuration file to the experiment direct- - ory..." + Copying the template NEMS configuration file to the experiment directory..." cp_vrfy "${NEMS_CONFIG_TMPL_FP}" "${NEMS_CONFIG_FP}" # # Copy the CCPP physics suite definition file from its location in the diff --git a/ush/setup.sh b/ush/setup.sh index c68e2cbeda..ee49750ae9 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1013,6 +1013,8 @@ case "$MACHINE" in "WCOSS_CRAY") FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} + FIXaer=${FIXaer:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} + FIXlut=${FIXlut:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1020,6 +1022,8 @@ case "$MACHINE" in "WCOSS_DELL_P3") FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} + FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} + FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1027,6 +1031,8 @@ case "$MACHINE" in "HERA") FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} + FIXaer=${FIXaer:-"/scratch1/NCEPDEV/global/glopara/fix/fix_aer"} + FIXlut=${FIXlut:-"/scratch1/NCEPDEV/global/glopara/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} @@ -1034,6 +1040,8 @@ case "$MACHINE" in "ORION") FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} + FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} + FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1041,6 +1049,8 @@ case "$MACHINE" in "JET") FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} + FIXaer=${FIXaer:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_aer"} + FIXlut=${FIXlut:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1048,6 +1058,8 @@ case "$MACHINE" in "ODIN") FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} + FIXaer=${FIXaer:-"/scratch/ywang/fix/theia_fix/fix_aer"} + FIXlut=${FIXlut:-"/scratch/ywang/fix/theia_fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1055,6 +1067,8 @@ case "$MACHINE" in "CHEYENNE") FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} + FIXaer=${FIXaer:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_aer"} + FIXlut=${FIXlut:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} @@ -1062,17 +1076,21 @@ case "$MACHINE" in "STAMPEDE") FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} + FIXaer=${FIXaer:-"/work/00315/tg455890/stampede2/regional_fv3/fix_aer"} + FIXlut=${FIXlut:-"/work/00315/tg455890/stampede2/regional_fv3/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} ;; *) - if [ -z "$FIXgsm" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then + if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} + FIXaer = \"${FIXaer:-\"\"} + FIXlut = \"${FIXlut:-\"\"} TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} @@ -1473,6 +1491,10 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # the fixed files containing various fields on global grids (which are # usually much coarser than the native FV3-LAM grid). # +# FIXclim: +# This is the directory that will contain the MERRA2 aerosol climatology +# data file and lookup tables for optics properties +# # FIXLAM: # This is the directory that will contain the fixed files or symlinks to # the fixed files containing the grid, orography, and surface climatology @@ -1507,6 +1529,7 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" LOGDIR="${EXPTDIR}/log" FIXam="${EXPTDIR}/fix_am" +FIXclim="${EXPTDIR}/fix_clim" FIXLAM="${EXPTDIR}/fix_lam" if [ "${RUN_ENVIR}" = "nco" ]; then @@ -2857,8 +2880,11 @@ PARMDIR="$PARMDIR" MODULES_DIR="${MODULES_DIR}" EXECDIR="$EXECDIR" FIXam="$FIXam" +FIXclim="$FIXclim" FIXLAM="$FIXLAM" FIXgsm="$FIXgsm" +FIXaer="$FIXaer" +FIXlut="$FIXlut" COMROOT="$COMROOT" COMOUT_BASEDIR="${COMOUT_BASEDIR}" TEMPLATE_DIR="${TEMPLATE_DIR}" diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index ce8f833f5b..b971671dd0 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -62,8 +62,6 @@ FV3_HRRR: kord_wz: 9 nord_tr: 2 nrows_blend: 10 - regional_bcs_from_gsi: False - write_restart_with_bcs: False gfs_physics_nml: <<: *gsd_sar_phys cdmbgwd: [3.5, 1.0] @@ -287,6 +285,52 @@ FV3_GFS_v15p2: ldebug: False surf_map_nml: +FV3_GFS_v15_thompson_mynn_lam3km: + atmos_model_nml: + avg_max_length: 3600.0 + fv_core_nml: + agrid_vel_rst: true + full_zs_filter: !!python/none + n_sponge: 9 + npz_type: '' + rf_fast: false + sg_cutoff: 10000.0 + vtdm4: 0.02 + gfs_physics_nml: + avg_max_length: 3600.0 + cdmbgwd: [0.88, 0.04] + do_deep: false + do_mynnsfclay: true + do_ugwp: false + fhswr: 900.0 + fhlwr: 900.0 + iaer: 1011 + iccn: 2 + icliq_sw: 2 + imfdeepcnv: 2 + imfshalcnv: 2 + iopt_alb: 2 + iopt_btr: 1 + iopt_crs: 1 + iopt_dveg: 2 + iopt_frz: 1 + iopt_inf: 1 + iopt_rad: 1 + iopt_run: 1 + iopt_sfc: 1 + iopt_snf: 4 + iopt_stc: 1 + iopt_tbot: 2 + iovr: 3 + ldiag_ugwp: false + lgfdlmprad: false + lsm: 1 + lsoil_lsm: !!python/none + ltaerosol: false + xkzminv: 0.3 + xkzm_m: 1.0 + xkzm_h: 1.0 + FV3_GFS_v16: cires_ugwp_nml: launch_level: 27 diff --git a/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km b/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km new file mode 100644 index 0000000000..84acd3caf5 --- /dev/null +++ b/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km @@ -0,0 +1,340 @@ +{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional +{{ starttime.strftime("%Y %m %d %H %M %S") }} + +#output files +"grid_spec", -1, "months", 1, "days", "time" +#"atmos_4xdaily", 6, "hours", 1, "days", "time" +"atmos_static", -1, "hours", 1, "hours", "time" +"fv3_history", 3, "hours", 1, "hours", "time" +"fv3_history2d", 3, "hours", 1, "hours", "time" + +# +#======================= +# ATMOSPHERE DIAGNOSTICS +#======================= +### +# grid_spec +### + "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, + "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, + "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, +### +# 4x daily output +### +# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 +#### +#"dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 +#### +# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 +# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 +### +# gfs static data +### + "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 + "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 + "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 + "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 + "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 +### +# FV3 variabls needed for NGGPS evaluation +### +"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "q_rimef", "q_rimef", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "pfnh", "pfnh", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 +#"gfs_dyn", "cld_amt", "cld_amt", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "ustm", "ustm", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "vstm", "vstm", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "srh01", "srh01", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "srh03", "srh03", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "u10max", "u10max", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10max", "v10max", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spd10max", "spd10max", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "refdmax", "refdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "refdmax263k","refdmax263k","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "t02max", "t02max", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "t02min", "t02min", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rh02max", "rh02max", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "rh02min", "rh02min", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "refl_10cm", "refl_10cm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cldfra", "cldfra", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pratemax", "pratemax", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 +#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 + +"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 +# Aerosols (CCN, IN) from Thompson microphysics +#"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 +#"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 +"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 +"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 + +#============================================================================================= +# +#====> This file can be used with diag_manager/v2.0a (or higher) <==== +# +# +# FORMATS FOR FILE ENTRIES (not all input values are used) +# ------------------------ +# +#"file_name", output_freq, "output_units", format, "time_units", "long_name", +# +# +#output_freq: > 0 output frequency in "output_units" +# = 0 output frequency every time step +# =-1 output frequency at end of run +# +#output_units = units used for output frequency +# (years, months, days, minutes, hours, seconds) +# +#time_units = units used to label the time axis +# (days, minutes, hours, seconds) +# +# +# FORMAT FOR FIELD ENTRIES (not all input values are used) +# ------------------------ +# +#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing +# +#time_avg = .true. or .false. +# +#packing = 1 double precision +# = 2 float +# = 4 packed 16-bit integers +# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km b/ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km new file mode 100644 index 0000000000..6fea174372 --- /dev/null +++ b/ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km @@ -0,0 +1,51 @@ +# added by FRE: sphum must be present in atmos +# specific humidity for moist runs + "TRACER", "atmos_mod", "sphum" + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=3.e-6" / +# prognostic cloud water mixing ratio + "TRACER", "atmos_mod", "liq_wat" + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic ice water mixing ratio + "TRACER", "atmos_mod", "ice_wat" + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic rain water mixing ratio + "TRACER", "atmos_mod", "rainwat" + "longname", "rain water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic snow water mixing ratio + "TRACER", "atmos_mod", "snowwat" + "longname", "snow water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic Grau water mixing ratio + "TRACER", "atmos_mod", "graupel" + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic cloud ice number concentration + "TRACER", "atmos_mod", "ice_nc" + "longname", "cloud ice water number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic rain number concentration + "TRACER", "atmos_mod", "rain_nc" + "longname", "rain number concentration" + "units", "/kg" + "profile_type", "fixed", "surface_value=0.0" / +# prognostic ozone mixing ratio tracer + "TRACER", "atmos_mod", "o3mr" + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 2d35ca4770..645e216011 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -113,6 +113,7 @@ range_warn = .true. read_increment = .false. regional = .true. + regional_bcs_from_gsi = .false. res_latlon_dynamics = 'fv3_increment.nc' reset_eta = .false. rf_cutoff = 20.e2 @@ -120,6 +121,7 @@ use_hydro_pressure = .false. vtdm4 = 0.075 warm_start = .false. + write_restart_with_bcs = .false. z_tracer = .true. / diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 8104038da9..a8bd0ae721 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -35,6 +35,7 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GSD_SAR" \ "FV3_GSD_v0" \ "FV3_GFS_v15p2" \ +"FV3_GFS_v15_thompson_mynn_lam3km" \ "FV3_GFS_v16" \ "FV3_RRFS_v1beta" \ "FV3_RRFS_v1alpha" \ From a429a479948ac8e166202e0bb0d67dde00184b62 Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Tue, 21 Dec 2021 08:48:58 -0700 Subject: [PATCH 108/203] Adding variables, levels, and thresholds to ensemble vx (feature/add_ensvx_vars) (#649) * Modifications to grid-to-point ensemble vx for additional variables, levels, and thresholds. * Update ensemble thresholds for APCP. Turning on OBS ERR for APCP. * Turn off OBS ERR for point * Turn OBS ERR to false in EnsembleStatConfig_point * Turned on obs error for ensemble point vx; fixed TCDC fcst options in conf files. * Broke out BOTH_ to FCST_ and OBS_ in certain METplus conf files. * Fixed typo in GridStat_APCP06h_prob.conf. * Add NBR ens probability section to APCP and REFC ensemble-stat configs. * Addressed typo in EnsembleStatConfig_APCP. * Removed double TCDC entry in PointStat_conus_sfc_prob.conf. Co-authored-by: Jamie Wolff --- .../parm/met/EnsembleStatConfig_APCP | 35 ++- .../parm/met/EnsembleStatConfig_REFC | 33 +- .../parm/metplus/EnsembleStat_APCP01h.conf | 4 +- .../parm/metplus/EnsembleStat_APCP03h.conf | 2 +- .../parm/metplus/EnsembleStat_APCP06h.conf | 2 +- .../parm/metplus/EnsembleStat_APCP24h.conf | 2 +- .../parm/metplus/EnsembleStat_conus_sfc.conf | 80 ++++- .../parm/metplus/EnsembleStat_upper_air.conf | 162 ++++++++-- .../parm/metplus/GridStat_APCP01h_mean.conf | 2 +- .../parm/metplus/GridStat_APCP01h_prob.conf | 12 +- .../parm/metplus/GridStat_APCP03h_mean.conf | 2 +- .../parm/metplus/GridStat_APCP03h_prob.conf | 37 +-- .../parm/metplus/GridStat_APCP06h_mean.conf | 2 +- .../parm/metplus/GridStat_APCP06h_prob.conf | 39 +-- .../parm/metplus/GridStat_APCP24h_mean.conf | 2 +- .../parm/metplus/GridStat_APCP24h_prob.conf | 39 +-- .../parm/metplus/PointStat_conus_sfc.conf | 9 +- .../metplus/PointStat_conus_sfc_mean.conf | 12 +- .../metplus/PointStat_conus_sfc_prob.conf | 182 +++++++++-- .../metplus/PointStat_upper_air_mean.conf | 105 ++++++- .../metplus/PointStat_upper_air_prob.conf | 295 ++++++++++++++++-- 21 files changed, 833 insertions(+), 225 deletions(-) diff --git a/ush/templates/parm/met/EnsembleStatConfig_APCP b/ush/templates/parm/met/EnsembleStatConfig_APCP index 34d560cb6a..f9569105aa 100755 --- a/ush/templates/parm/met/EnsembleStatConfig_APCP +++ b/ush/templates/parm/met/EnsembleStatConfig_APCP @@ -59,6 +59,33 @@ ens = { //////////////////////////////////////////////////////////////////////////////// +// +// Neighborhood ensemble probabilities +// +nbrhd_prob = { + width = [ 5 ]; + shape = CIRCLE; + vld_thresh = 0.0; +} + +// +// NMEP smoothing methods +// +nmep_smooth = { + vld_thresh = 0.0; + shape = CIRCLE; + gaussian_dx = 81.27; + gaussian_radius = 120; + type = [ + { + method = GAUSSIAN; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + // // Forecast and observation fields to be verified // @@ -91,7 +118,7 @@ skip_const = TRUE; // May be set separately in each "obs.field" entry // obs_error = { - flag = FALSE; // TRUE or FALSE + flag = TRUE; // TRUE or FALSE dist_type = NONE; // Distribution type dist_parm = []; // Distribution parameters inst_bias_scale = 1.0; // Instrument bias scale adjustment @@ -207,11 +234,11 @@ interp = { // Statistical output types // output_flag = { - ecnt = NONE; + ecnt = STAT; rps = NONE; rhist = STAT; phist = STAT; - orank = NONE; + orank = STAT; ssvar = STAT; relp = STAT; } @@ -234,7 +261,7 @@ ensemble_flag = { frequency = TRUE; nep = FALSE; nmep = FALSE; - rank = FALSE; + rank = TRUE; weight = FALSE; } diff --git a/ush/templates/parm/met/EnsembleStatConfig_REFC b/ush/templates/parm/met/EnsembleStatConfig_REFC index 5d44356b63..385e01a987 100755 --- a/ush/templates/parm/met/EnsembleStatConfig_REFC +++ b/ush/templates/parm/met/EnsembleStatConfig_REFC @@ -63,6 +63,33 @@ ens = { //////////////////////////////////////////////////////////////////////////////// +// +// Neighborhood ensemble probabilities +// +nbrhd_prob = { + width = [ 5 ]; + shape = CIRCLE; + vld_thresh = 0.0; +} + +// +// NMEP smoothing methods +// +nmep_smooth = { + vld_thresh = 0.0; + shape = CIRCLE; + gaussian_dx = 81.27; + gaussian_radius = 120; + type = [ + { + method = GAUSSIAN; + width = 1; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + // // Forecast and observation fields to be verified // @@ -215,11 +242,11 @@ interp = { // Statistical output types // output_flag = { - ecnt = NONE; + ecnt = STAT; rps = NONE; rhist = STAT; phist = STAT; - orank = NONE; + orank = STAT; ssvar = STAT; relp = STAT; } @@ -242,7 +269,7 @@ ensemble_flag = { frequency = TRUE; nep = FALSE; nmep = FALSE; - rank = FALSE; + rank = TRUE; weight = FALSE; } diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf index 46dd14a82e..9172b3c17b 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf @@ -101,14 +101,14 @@ ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. -ENSEMBLE_STAT_MET_OBS_ERR_TABLE = +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A01 -ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 +ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge2.54 # Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf index cf7c95a23f..fbe1f518ae 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf @@ -150,7 +150,7 @@ ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A03 -ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +ENS_VAR1_THRESH = gt0.0,ge0.508,ge2.54,ge6.350 # Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf index 9f8ddaafd6..168cb9b984 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf @@ -150,7 +150,7 @@ ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A06 -ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +ENS_VAR1_THRESH = gt0.0,ge2.54,ge6.350,ge12.700 # Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf index 4ce438fbb9..1243e1f54a 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf @@ -150,7 +150,7 @@ ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A24 -ENS_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +ENS_VAR1_THRESH = gt0.0,ge6.350,ge12.700,ge25.400 # Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index 09d2572d72..2523fde2a3 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -143,28 +143,78 @@ ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt ENS_VAR1_NAME = TMP ENS_VAR1_LEVELS = Z02 -ENS_VAR1_THRESH = >=293, >=298, >=303 +ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 ENS_VAR2_NAME = DPT ENS_VAR2_LEVELS = Z2 -ENS_VAR2_THRESH = >=288, >=293, >=298 +ENS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 ENS_VAR3_NAME = WIND ENS_VAR3_LEVELS = Z10 -ENS_VAR3_THRESH = >=5, >=10 +ENS_VAR3_THRESH = >=5, >=10, >=15 ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind -BOTH_VAR1_NAME = TMP -BOTH_VAR1_LEVELS = Z2 -BOTH_VAR1_THRESH = >=293, >=298, >=303 - -BOTH_VAR2_NAME = DPT -BOTH_VAR2_LEVELS = Z2 -BOTH_VAR2_THRESH = >=288, >=293, >=298 - -BOTH_VAR3_NAME = WIND -BOTH_VAR3_LEVELS = Z10 -BOTH_VAR3_THRESH = >=5, >=10 -BOTH_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind +ENS_VAR4_NAME = TCDC +ENS_VAR4_LEVELS = L0 +ENS_VAR4_THRESH = <25, >75 +ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = [ 14 ]; GRIB2_ipdtmpl_val = [ 0 ]; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR5_NAME = VIS +ENS_VAR5_LEVELS = L0 +ENS_VAR5_THRESH = <1609, <8045, >=8045 +ENS_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +ENS_VAR6_NAME = HGT +ENS_VAR6_LEVELS = L0 +ENS_VAR6_THRESH = <152, <1520, >=914 +ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; + +FCST_VAR1_NAME = TMP +FCST_VAR1_LEVELS = Z2 +FCST_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = Z2 +OBS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 +OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR2_NAME = DPT +FCST_VAR2_LEVELS = Z2 +FCST_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 +OBS_VAR2_NAME = DPT +OBS_VAR2_LEVELS = Z2 +OBS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 +OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR3_NAME = WIND +FCST_VAR3_LEVELS = Z10 +FCST_VAR3_THRESH = >=5, >=10, >=15 +FCST_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind +OBS_VAR3_NAME = WIND +OBS_VAR3_LEVELS = Z10 +OBS_VAR3_THRESH = >=5, >=10, >=15 +OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR4_NAME = TCDC +FCST_VAR4_LEVELS = L0 +FCST_VAR4_THRESH = <25, >75 +FCST_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = 8; GRIB2_ipdtmpl_val = {lead?fmt=%H}; +OBS_VAR4_NAME = TCDC +OBS_VAR4_LEVELS = L0 +OBS_VAR4_THRESH = <25, >75 +OBS_VAR4_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +BOTH_VAR5_NAME = VIS +BOTH_VAR5_LEVELS = L0 +BOTH_VAR5_THRESH = <1609, <8045, >=8045 +BOTH_VAR5_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR6_NAME = HGT +FCST_VAR6_LEVELS = L0 +FCST_VAR6_THRESH = <152, <1520, >=914 +FCST_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; +OBS_VAR6_NAME = CEILING +OBS_VAR6_LEVELS = L0 +OBS_VAR6_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } +OBS_VAR6_THRESH = <152, <305, >=914 ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf index 19d982ecd4..fb2070e87e 100644 --- a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -108,7 +108,7 @@ OBS_ENSEMBLE_STAT_WINDOW_END = {OBS_WINDOW_END} # number of expected members for ensemble. Should correspond with the # number of items in the list for FCST_ENSEMBLE_STAT_INPUT_TEMPLATE -ENSEMBLE_STAT_N_MEMBERS = 10 +ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # ens.ens_thresh value in the MET config file # threshold for ratio of valid files to expected files to allow app to run @@ -150,24 +150,146 @@ ENS_VAR1_NAME = TMP ENS_VAR1_LEVELS = P850 ENS_VAR1_THRESH = >=288, >=293, >=298 -ENS_VAR2_NAME = DPT -ENS_VAR2_LEVELS = P850 -ENS_VAR2_THRESH = >=283, >=288, >=293 - -ENS_VAR3_NAME = WIND -ENS_VAR3_LEVELS = P850 -ENS_VAR3_THRESH = >=5, >=10 - -BOTH_VAR1_NAME = TMP -BOTH_VAR1_LEVELS = P850 -BOTH_VAR1_THRESH = >=288, >=293, >=298 - -BOTH_VAR2_NAME = DPT -BOTH_VAR2_LEVELS = P850 -BOTH_VAR2_THRESH = >=283, >=288, >=293 - -BOTH_VAR3_NAME = WIND -BOTH_VAR3_LEVELS = P850 -BOTH_VAR3_THRESH = >=5, >=10 +ENS_VAR2_NAME = TMP +ENS_VAR2_LEVELS = P700 +ENS_VAR2_THRESH = >=273, >=278, >=283 + +ENS_VAR3_NAME = TMP +ENS_VAR3_LEVELS = P500 +ENS_VAR3_THRESH = >=258, >=263, >=268 + +ENS_VAR4_NAME = DPT +ENS_VAR4_LEVELS = P850 +ENS_VAR4_THRESH = >=273, >=278, >=283 + +ENS_VAR5_NAME = DPT +ENS_VAR5_LEVELS = P700 +ENS_VAR5_THRESH = >=263, >=268, >=273 + +ENS_VAR6_NAME = WIND +ENS_VAR6_LEVELS = P850 +ENS_VAR6_THRESH = >=5, >=10, >=15 + +ENS_VAR7_NAME = WIND +ENS_VAR7_LEVELS = P700 +ENS_VAR7_THRESH = >=10, >=15, >=20 + +ENS_VAR8_NAME = WIND +ENS_VAR8_LEVELS = P500 +ENS_VAR8_THRESH = >=15, >=21, >=26 + +ENS_VAR9_NAME = WIND +ENS_VAR9_LEVELS = P250 +ENS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 + +ENS_VAR10_NAME = HGT +ENS_VAR10_LEVELS = P500 +ENS_VAR10_THRESH = >=5400, >=5600, >=5880 + +ENS_VAR11_NAME = CAPE +ENS_VAR11_LEVELS = L0 +ENS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; +ENS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 + +ENS_VAR12_NAME = HPBL +ENS_VAR12_LEVELS = Z0 +ENS_VAR12_THRESH = <500, <1500, >1500 + +FCST_VAR1_NAME = TMP +FCST_VAR1_LEVELS = P850 +FCST_VAR1_THRESH = >=288, >=293, >=298 +OBS_VAR1_NAME = TMP +OBS_VAR1_LEVELS = P850 +OBS_VAR1_THRESH = >=288, >=293, >=298 +OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR2_NAME = TMP +FCST_VAR2_LEVELS = P700 +FCST_VAR2_THRESH = >=273, >=278, >=283 +OBS_VAR2_NAME = TMP +OBS_VAR2_LEVELS = P700 +OBS_VAR2_THRESH = >=273, >=278, >=283 +OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR3_NAME = TMP +FCST_VAR3_LEVELS = P500 +FCST_VAR3_THRESH = >=258, >=263, >=268 +OBS_VAR3_NAME = TMP +OBS_VAR3_LEVELS = P500 +OBS_VAR3_THRESH = >=258, >=263, >=268 +OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR4_NAME = DPT +FCST_VAR4_LEVELS = P850 +FCST_VAR4_THRESH = >=273, >=278, >=283 +OBS_VAR4_NAME = DPT +OBS_VAR4_LEVELS = P850 +OBS_VAR4_THRESH = >=273, >=278, >=283 +OBS_VAR4_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR5_NAME = DPT +FCST_VAR5_LEVELS = P700 +FCST_VAR5_THRESH = >=263, >=286, >=273 +OBS_VAR5_NAME = DPT +OBS_VAR5_LEVELS = P700 +OBS_VAR5_THRESH = >=263, >=286, >=273 +OBS_VAR5_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR6_NAME = WIND +FCST_VAR6_LEVELS = P850 +FCST_VAR6_THRESH = >=5, >=10, >=15 +OBS_VAR6_NAME = WIND +OBS_VAR6_LEVELS = P850 +OBS_VAR6_THRESH = >=5, >=10, >=15 +OBS_VAR6_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR7_NAME = WIND +FCST_VAR7_LEVELS = P700 +FCST_VAR7_THRESH = >=10, >=15, >=20 +OBS_VAR7_NAME = WIND +OBS_VAR7_LEVELS = P700 +OBSVAR7_THRESH = >=10, >=15, >=20 +OBS_VAR7_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR8_NAME = WIND +FCST_VAR8_LEVELS = P500 +FCST_VAR8_THRESH = >=15, >=21, >=26 +OBS_VAR8_NAME = WIND +OBS_VAR8_LEVELS = P500 +OBS_VAR8_THRESH = >=15, >=21, >=26 +OBS_VAR8_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR9_NAME = WIND +FCST_VAR9_LEVELS = P250 +FCST_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 +OBS_VAR9_NAME = WIND +OBS_VAR9_LEVELS = P250 +OBS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 +OBS_VAR9_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR10_NAME = HGT +FCST_VAR10_LEVELS = P500 +FCST_VAR10_THRESH = >=5400, >=5600, >=5880 +OBS_VAR10_NAME = HGT +OBS_VAR10_LEVELS = P500 +OBS_VAR10_THRESH = >=5400, >=5600, >=5880 +OBS_VAR10_OPTIONS = obs_error = { flag = TRUE; } + +FCST_VAR11_NAME = CAPE +FCST_VAR11_LEVELS = L0 +FCST_VAR11_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR11_THRESH = <=1000, >1000&&<2500, >=2500&&<4000, >=2500 +OBS_VAR11_NAME = CAPE +OBS_VAR11_LEVELS = L0-100000 +OBS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR11_THRESH = <=1000, >1000&&<2500, >=2500&&<4000, >=2500 + +FCST_VAR12_NAME = HPBL +FCST_VAR12_LEVELS = Z0 +FCST_VAR12_THRESH = <500, <1500, >1500 +OBS_VAR12_NAME = PBL +OBS_VAR12_LEVELS = L0 +OBS_VAR12_OPTIONS = desc = "TKE"; +OBS_VAR12_THRESH = <500, <1500, >1500 ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf index 16cd58e7fb..c43d63ead6 100644 --- a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf @@ -81,7 +81,7 @@ GRID_STAT_REGRID_TO_GRID = FCST FCST_VAR1_NAME = APCP_01_A01_ENS_MEAN FCST_VAR1_LEVELS = A01 -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge2.54 OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A01 diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf index 81b3d9a6d6..52065ba7a5 100644 --- a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf @@ -103,21 +103,13 @@ OBS_VAR3_NAME = APCP OBS_VAR3_LEVELS = A01 OBS_VAR3_THRESH = >=0.508 -FCST_VAR4_NAME = APCP_01_A01_ENS_FREQ_ge1.27 +FCST_VAR4_NAME = APCP_01_A01_ENS_FREQ_ge2.54 FCST_VAR4_LEVELS = A01 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A01 -OBS_VAR4_THRESH = >=1.27 - -FCST_VAR5_NAME = APCP_01_A01_ENS_FREQ_ge2.54 -FCST_VAR5_LEVELS = A01 -FCST_VAR5_THRESH = ==0.1 - -OBS_VAR5_NAME = APCP -OBS_VAR5_LEVELS = A01 -OBS_VAR5_THRESH = >=2.54 +OBS_VAR4_THRESH = >=2.54 # Neighborhood shape and widths diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf index 536ce241f4..67db9927d2 100644 --- a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf @@ -75,7 +75,7 @@ GRID_STAT_REGRID_TO_GRID = FCST FCST_VAR1_NAME = APCP_A3_ENS_MEAN FCST_VAR1_LEVELS = A3 -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +BOTH_VAR1_THRESH = gt0.0,ge0.508,ge2.54,ge6.350 OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A3 diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf index 984eba7f01..90f16b5223 100644 --- a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf @@ -80,54 +80,29 @@ OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A03 OBS_VAR1_THRESH = >0.0 -FCST_VAR2_NAME = APCP_A3_ENS_FREQ_ge0.254 +FCST_VAR2_NAME = APCP_A3_ENS_FREQ_ge0.508 FCST_VAR2_LEVELS = A03 FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = APCP OBS_VAR2_LEVELS = A03 -OBS_VAR2_THRESH = >=0.254 +OBS_VAR2_THRESH = >=0.508 -FCST_VAR3_NAME = APCP_A3_ENS_FREQ_ge0.508 +FCST_VAR3_NAME = APCP_A3_ENS_FREQ_ge2.54 FCST_VAR3_LEVELS = A03 FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = APCP OBS_VAR3_LEVELS = A03 -OBS_VAR3_THRESH = >=0.508 +OBS_VAR3_THRESH = >=2.54 -FCST_VAR4_NAME = APCP_A3_ENS_FREQ_ge1.27 +FCST_VAR4_NAME = APCP_A3_ENS_FREQ_ge6.350 FCST_VAR4_LEVELS = A03 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A03 -OBS_VAR4_THRESH = >=1.27 - -FCST_VAR5_NAME = APCP_A3_ENS_FREQ_ge2.54 -FCST_VAR5_LEVELS = A03 -FCST_VAR5_THRESH = ==0.1 - -OBS_VAR5_NAME = APCP -OBS_VAR5_LEVELS = A03 -OBS_VAR5_THRESH = >=2.54 - -FCST_VAR6_NAME = APCP_A3_ENS_FREQ_ge3.810 -FCST_VAR6_LEVELS = A03 -FCST_VAR6_THRESH = ==0.1 - -OBS_VAR6_NAME = APCP -OBS_VAR6_LEVELS = A03 -OBS_VAR6_THRESH = >=3.810 - -FCST_VAR7_NAME = APCP_A3_ENS_FREQ_ge6.350 -FCST_VAR7_LEVELS = A03 -FCST_VAR7_THRESH = ==0.1 - -OBS_VAR7_NAME = APCP -OBS_VAR7_LEVELS = A03 -OBS_VAR7_THRESH = >=6.350 - +OBS_VAR4_THRESH = >=6.350 # Neighborhood shape and widths GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf index b10707f0fa..ef2ddb9535 100644 --- a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf @@ -75,7 +75,7 @@ GRID_STAT_REGRID_TO_GRID = FCST FCST_VAR1_NAME = APCP_A6_ENS_MEAN FCST_VAR1_LEVELS = A06 -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +BOTH_VAR1_THRESH = gt0.0,ge2.54,ge6.350,ge12.700 OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A06 diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf index 7c5e086721..5f75e50975 100644 --- a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf @@ -80,54 +80,29 @@ OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A06 OBS_VAR1_THRESH = >0.0 -FCST_VAR2_NAME = APCP_A6_ENS_FREQ_ge0.254 +FCST_VAR2_NAME = APCP_A6_ENS_FREQ_ge2.54 FCST_VAR2_LEVELS = A06 FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = APCP OBS_VAR2_LEVELS = A06 -OBS_VAR2_THRESH = >=0.254 +OBS_VAR2_THRESH = >=2.54 -FCST_VAR3_NAME = APCP_A6_ENS_FREQ_ge0.508 +FCST_VAR3_NAME = APCP_A6_ENS_FREQ_ge6.350 FCST_VAR3_LEVELS = A06 FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = APCP OBS_VAR3_LEVELS = A06 -OBS_VAR3_THRESH = >=0.508 +OBS_VAR3_THRESH = >=6.350 -FCST_VAR4_NAME = APCP_A6_ENS_FREQ_ge1.27 -FCST_VAR4_LEVELS = A06 +FCST_VAR4_NAME = APCP_A6_ENS_FREQ_ge12.700 +FCST_VAR4_LEVELS = A06 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A06 -OBS_VAR4_THRESH = >=1.27 - -FCST_VAR5_NAME = APCP_A6_ENS_FREQ_ge2.54 -FCST_VAR5_LEVELS = A06 -FCST_VAR5_THRESH = ==0.1 - -OBS_VAR5_NAME = APCP -OBS_VAR5_LEVELS = A06 -OBS_VAR5_THRESH = >=2.54 - -FCST_VAR6_NAME = APCP_A6_ENS_FREQ_ge3.810 -FCST_VAR6_LEVELS = A06 -FCST_VAR6_THRESH = ==0.1 - -OBS_VAR6_NAME = APCP -OBS_VAR6_LEVELS = A06 -OBS_VAR6_THRESH = >=3.810 - -FCST_VAR7_NAME = APCP_A6_ENS_FREQ_ge6.350 -FCST_VAR7_LEVELS = A06 -FCST_VAR7_THRESH = ==0.1 - -OBS_VAR7_NAME = APCP -OBS_VAR7_LEVELS = A06 -OBS_VAR7_THRESH = >=6.350 - +OBS_VAR4_THRESH = >=12.700 # Neighborhood shape and widths GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf index aa0c53e210..ad1308e20f 100644 --- a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf @@ -75,7 +75,7 @@ GRID_STAT_REGRID_TO_GRID = FCST FCST_VAR1_NAME = APCP_A24_ENS_MEAN FCST_VAR1_LEVELS = A24 -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 +BOTH_VAR1_THRESH = gt0.0,ge6.350,ge12.700,ge25.400 OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A24 diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf index 26d6e6368b..a3659ea7b9 100644 --- a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf @@ -80,54 +80,29 @@ OBS_VAR1_NAME = APCP OBS_VAR1_LEVELS = A24 OBS_VAR1_THRESH = >0.0 -FCST_VAR2_NAME = APCP_A24_ENS_FREQ_ge0.254 -FCST_VAR2_LEVELS = A24 +FCST_VAR2_NAME = APCP_A24_ENS_FREQ_ge6.350 +FCST_VAR2_LEVELS = A24 FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = APCP OBS_VAR2_LEVELS = A24 -OBS_VAR2_THRESH = >=0.254 +OBS_VAR2_THRESH = >=6.350 -FCST_VAR3_NAME = APCP_A24_ENS_FREQ_ge0.508 +FCST_VAR3_NAME = APCP_A24_ENS_FREQ_ge12.700 FCST_VAR3_LEVELS = A24 FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = APCP OBS_VAR3_LEVELS = A24 -OBS_VAR3_THRESH = >=0.508 +OBS_VAR3_THRESH = >=12.700 -FCST_VAR4_NAME = APCP_A24_ENS_FREQ_ge1.27 +FCST_VAR4_NAME = APCP_A24_ENS_FREQ_ge25.400 FCST_VAR4_LEVELS = A24 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A24 -OBS_VAR4_THRESH = >=1.27 - -FCST_VAR5_NAME = APCP_A24_ENS_FREQ_ge2.54 -FCST_VAR5_LEVELS = A24 -FCST_VAR5_THRESH = ==0.1 - -OBS_VAR5_NAME = APCP -OBS_VAR5_LEVELS = A24 -OBS_VAR5_THRESH = >=2.54 - -FCST_VAR6_NAME = APCP_A24_ENS_FREQ_ge3.810 -FCST_VAR6_LEVELS = A24 -FCST_VAR6_THRESH = ==0.1 - -OBS_VAR6_NAME = APCP -OBS_VAR6_LEVELS = A24 -OBS_VAR6_THRESH = >=3.810 - -FCST_VAR7_NAME = APCP_A24_ENS_FREQ_ge6.350 -FCST_VAR7_LEVELS = A24 -FCST_VAR7_THRESH = ==0.1 - -OBS_VAR7_NAME = APCP -OBS_VAR7_LEVELS = A24 -OBS_VAR7_THRESH = >=6.350 - +OBS_VAR4_THRESH = >=25.400 # Neighborhood shape and widths GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 5fd3f58c9b..4eccff4c18 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -153,9 +153,12 @@ BOTH_VAR6_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V co BOTH_VAR7_NAME = PRMSL BOTH_VAR7_LEVELS = Z0 -BOTH_VAR8_NAME = TCDC -BOTH_VAR8_LEVELS = L0 -BOTH_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } +FCST_VAR8_NAME = TCDC +FCST_VAR8_LEVELS = L0 +FCST_VAR8_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = 8; GRIB2_ipdtmpl_val = {lead?fmt=%H}; +OBS_VAR8_NAME = TCDC +OBS_VAR8_LEVELS = L0 +OBS_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } BOTH_VAR9_NAME = VIS BOTH_VAR9_LEVELS = L0 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf index c5adc81df5..fcc45a4c9a 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf @@ -128,24 +128,24 @@ POINT_STAT_MESSAGE_TYPE = ADPSFC # (optional) FCST_VARn_OPTION FCST_VAR1_NAME = TMP_Z2_ENS_MEAN FCST_VAR1_LEVELS = Z2 -FCST_VAR1_THRESH = >=293, >=298, >=303 +FCST_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 OBS_VAR1_NAME = TMP OBS_VAR1_LEVELS = Z2 -OBS_VAR1_THRESH = >=293, >=298, >=303 +OBS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 FCST_VAR2_NAME = DPT_Z2_ENS_MEAN FCST_VAR2_LEVELS = Z2 -FCST_VAR2_THRESH = >=288, >=293, >=298 +FCST_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 OBS_VAR2_NAME = DPT OBS_VAR2_LEVELS = Z2 -OBS_VAR2_THRESH = >=288, >=293, >=298 +OBS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 FCST_VAR3_NAME = WIND_Z10_ENS_MEAN FCST_VAR3_LEVELS = Z10 -FCST_VAR3_THRESH = >=5, >=10 +FCST_VAR3_THRESH = >=5, >=10, >=15 OBS_VAR3_NAME = WIND OBS_VAR3_LEVELS = Z10 -OBS_VAR3_THRESH = >=5, >=10 +OBS_VAR3_THRESH = >=5, >=10, >=15 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf index f7ef100f77..bffc327add 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -126,69 +126,201 @@ POINT_STAT_MESSAGE_TYPE = ADPSFC # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION -FCST_VAR1_NAME = TMP_Z2_ENS_FREQ_ge293 +FCST_VAR1_NAME = TMP_Z2_ENS_FREQ_ge268 FCST_VAR1_LEVELS = (*,*) FCST_VAR1_THRESH = ==0.1 OBS_VAR1_NAME = TMP OBS_VAR1_LEVELS = Z2 -OBS_VAR1_THRESH = >=293 +OBS_VAR1_THRESH = >=268 -FCST_VAR2_NAME = TMP_Z2_ENS_FREQ_ge298 +FCST_VAR2_NAME = TMP_Z2_ENS_FREQ_ge273 FCST_VAR2_LEVELS = (*,*) FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = TMP OBS_VAR2_LEVELS = Z2 -OBS_VAR2_THRESH = >=298 +OBS_VAR2_THRESH = >=273 -FCST_VAR3_NAME = TMP_Z2_ENS_FREQ_ge303 +FCST_VAR3_NAME = TMP_Z2_ENS_FREQ_ge278 FCST_VAR3_LEVELS = (*,*) FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = TMP OBS_VAR3_LEVELS = Z2 -OBS_VAR3_THRESH = >=303 +OBS_VAR3_THRESH = >=278 -FCST_VAR4_NAME = DPT_Z2_ENS_FREQ_ge288 +FCST_VAR4_NAME = TMP_Z2_ENS_FREQ_ge293 FCST_VAR4_LEVELS = (*,*) FCST_VAR4_THRESH = ==0.1 -OBS_VAR4_NAME = DPT +OBS_VAR4_NAME = TMP OBS_VAR4_LEVELS = Z2 -OBS_VAR4_THRESH = >=288 +OBS_VAR4_THRESH = >=293 -FCST_VAR5_NAME = DPT_Z2_ENS_FREQ_ge293 +FCST_VAR5_NAME = TMP_Z2_ENS_FREQ_ge298 FCST_VAR5_LEVELS = (*,*) FCST_VAR5_THRESH = ==0.1 -OBS_VAR5_NAME = DPT +OBS_VAR5_NAME = TMP OBS_VAR5_LEVELS = Z2 -OBS_VAR5_THRESH = >=293 +OBS_VAR5_THRESH = >=298 -FCST_VAR6_NAME = DPT_Z2_ENS_FREQ_ge298 +FCST_VAR6_NAME = TMP_Z2_ENS_FREQ_ge303 FCST_VAR6_LEVELS = (*,*) FCST_VAR6_THRESH = ==0.1 -OBS_VAR6_NAME = DPT +OBS_VAR6_NAME = TMP OBS_VAR6_LEVELS = Z2 -OBS_VAR6_THRESH = >=298 +OBS_VAR6_THRESH = >=303 -FCST_VAR7_NAME = WIND_Z10_ENS_FREQ_ge5 -FCST_VAR7_LEVELS = Z10 +FCST_VAR7_NAME = DPT_Z2_ENS_FREQ_ge263 +FCST_VAR7_LEVELS = (*,*) FCST_VAR7_THRESH = ==0.1 -OBS_VAR7_NAME = WIND -OBS_VAR7_LEVELS = Z10 -OBS_VAR7_THRESH = >=5 +OBS_VAR7_NAME = DPT +OBS_VAR7_LEVELS = Z2 +OBS_VAR7_THRESH = >=263 -FCST_VAR8_NAME = WIND_Z10_ENS_FREQ_ge10 -FCST_VAR8_LEVELS = Z10 +FCST_VAR8_NAME = DPT_Z2_ENS_FREQ_ge268 +FCST_VAR8_LEVELS = (*,*) FCST_VAR8_THRESH = ==0.1 -OBS_VAR8_NAME = WIND -OBS_VAR8_LEVELS = Z10 -OBS_VAR8_THRESH = >=10 +OBS_VAR8_NAME = DPT +OBS_VAR8_LEVELS = Z2 +OBS_VAR8_THRESH = >=268 + +FCST_VAR9_NAME = DPT_Z2_ENS_FREQ_ge273 +FCST_VAR9_LEVELS = (*,*) +FCST_VAR9_THRESH = ==0.1 + +OBS_VAR9_NAME = DPT +OBS_VAR9_LEVELS = Z2 +OBS_VAR9_THRESH = >=273 + +FCST_VAR10_NAME = DPT_Z2_ENS_FREQ_ge288 +FCST_VAR10_LEVELS = (*,*) +FCST_VAR10_THRESH = ==0.1 + +OBS_VAR10_NAME = DPT +OBS_VAR10_LEVELS = Z2 +OBS_VAR10_THRESH = >=288 + +FCST_VAR11_NAME = DPT_Z2_ENS_FREQ_ge293 +FCST_VAR11_LEVELS = (*,*) +FCST_VAR11_THRESH = ==0.1 + +OBS_VAR11_NAME = DPT +OBS_VAR11_LEVELS = Z2 +OBS_VAR11_THRESH = >=293 + +FCST_VAR12_NAME = DPT_Z2_ENS_FREQ_ge298 +FCST_VAR12_LEVELS = (*,*) +FCST_VAR12_THRESH = ==0.1 + +OBS_VAR12_NAME = DPT +OBS_VAR12_LEVELS = Z2 +OBS_VAR12_THRESH = >=298 + +FCST_VAR13_NAME = WIND_Z10_ENS_FREQ_ge5 +FCST_VAR13_LEVELS = Z10 +FCST_VAR13_THRESH = ==0.1 + +OBS_VAR13_NAME = WIND +OBS_VAR13_LEVELS = Z10 +OBS_VAR13_THRESH = >=5 + +FCST_VAR14_NAME = WIND_Z10_ENS_FREQ_ge10 +FCST_VAR14_LEVELS = Z10 +FCST_VAR14_THRESH = ==0.1 + +OBS_VAR14_NAME = WIND +OBS_VAR14_LEVELS = Z10 +OBS_VAR14_THRESH = >=10 + +FCST_VAR15_NAME = WIND_Z10_ENS_FREQ_ge15 +FCST_VAR15_LEVELS = Z10 +FCST_VAR15_THRESH = ==0.1 + +OBS_VAR15_NAME = WIND +OBS_VAR15_LEVELS = Z10 +OBS_VAR15_THRESH = >=15 + +FCST_VAR16_NAME = TCDC_L0_ENS_FREQ_lt25 +FCST_VAR16_LEVELS = L0 +FCST_VAR16_THRESH = ==0.1 + +OBS_VAR16_NAME = TCDC +OBS_VAR16_LEVELS = L0 +OBS_VAR16_THRESH = <25 + +FCST_VAR17_NAME = TCDC_L0_ENS_FREQ_gt75 +FCST_VAR17_LEVELS = L0 +FCST_VAR17_THRESH = ==0.1 + +OBS_VAR17_NAME = TCDC +OBS_VAR17_LEVELS = L0 +OBS_VAR17_THRESH = >75 + +FCST_VAR18_NAME = VIS_L0_ENS_FREQ_lt1609 +FCST_VAR18_LEVELS = L0 +FCST_VAR18_THRESH = ==0.1 +FCST_VAR18_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +OBS_VAR18_NAME = VIS +OBS_VAR18_LEVELS = L0 +OBS_VAR18_THRESH = <1609 +OBS_VAR18_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR19_NAME = VIS_L0_ENS_FREQ_lt8045 +FCST_VAR19_LEVELS = L0 +FCST_VAR19_THRESH = ==0.1 +FCST_VAR19_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +OBS_VAR19_NAME = VIS +OBS_VAR19_LEVELS = L0 +OBS_VAR19_THRESH = <8045 +OBS_VAR19_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR20_NAME = VIS_L0_ENS_FREQ_ge8045 +FCST_VAR20_LEVELS = L0 +FCST_VAR20_THRESH = ==0.1 +FCST_VAR20_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +OBS_VAR20_NAME = VIS +OBS_VAR20_LEVELS = L0 +OBS_VAR20_THRESH = >=8045 +OBS_VAR20_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR21_NAME = HGT_L0_ENS_FREQ_lt152 +FCST_VAR21_LEVELS = L0 +FCST_VAR21_THRESH = ==0.1 +FCST_VAR21_OPTIONS = desc = "CEILING"; + +OBS_VAR21_NAME = CEILING +OBS_VAR21_LEVELS = L0 +OBS_VAR21_THRESH = <152 +OBS_VAR21_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR22_NAME = HGT_L0_ENS_FREQ_lt1520 +FCST_VAR22_LEVELS = L0 +FCST_VAR22_THRESH = ==0.1 +FCST_VAR22_OPTIONS = desc = "CEILING"; + +OBS_VAR22_NAME = CEILING +OBS_VAR22_LEVELS = L0 +OBS_VAR22_THRESH = <1520 +OBS_VAR22_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } + +FCST_VAR23_NAME = HGT_L0_ENS_FREQ_ge914 +FCST_VAR23_LEVELS = L0 +FCST_VAR23_THRESH = ==0.1 +FCST_VAR23_OPTIONS = desc = "CEILING"; + +OBS_VAR23_NAME = CEILING +OBS_VAR23_LEVELS = L0 +OBS_VAR23_THRESH = >=914 +OBS_VAR23_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } # Forecast data description variables FCST_IS_PROB = True diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf index e6a1a062ad..3f7dfc8f27 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -135,18 +135,93 @@ OBS_VAR1_NAME = TMP OBS_VAR1_LEVELS = P850 OBS_VAR1_THRESH = >=288, >=293, >=298 -FCST_VAR2_NAME = DPT_P850_ENS_MEAN -FCST_VAR2_LEVELS = P850 -FCST_VAR2_THRESH = >=283, >=288, >=293 - -OBS_VAR2_NAME = DPT -OBS_VAR2_LEVELS = P850 -OBS_VAR2_THRESH = >=283, >=288, >=293 - -FCST_VAR3_NAME = WIND_P850_ENS_MEAN -FCST_VAR3_LEVELS = P850 -FCST_VAR3_THRESH = >=5, >=10 - -OBS_VAR3_NAME = WIND -OBS_VAR3_LEVELS = P850 -OBS_VAR3_THRESH = >=5, >=10 +FCST_VAR2_NAME = TMP_P700_ENS_MEAN +FCST_VAR2_LEVELS = P700 +FCST_VAR2_THRESH = >=273, >=278, >=283 + +OBS_VAR2_NAME = TMP +OBS_VAR2_LEVELS = P700 +OBS_VAR2_THRESH = >=273, >=278, >=283 + +FCST_VAR3_NAME = TMP_P500_ENS_MEAN +FCST_VAR3_LEVELS = P500 +FCST_VAR3_THRESH = >=258, >=263, >=268 + +OBS_VAR3_NAME = TMP +OBS_VAR3_LEVELS = P500 +OBS_VAR3_THRESH = >=258, >=263, >=268 + +FCST_VAR4_NAME = DPT_P850_ENS_MEAN +FCST_VAR4_LEVELS = P850 +FCST_VAR4_THRESH = >=273, >=278, >=283 + +OBS_VAR4_NAME = DPT +OBS_VAR4_LEVELS = P850 +OBS_VAR4_THRESH = >=273, >=278, >=283 + +FCST_VAR5_NAME = DPT_P850_ENS_MEAN +FCST_VAR5_LEVELS = P700 +FCST_VAR5_THRESH = >=263, >=286, >=273 + +OBS_VAR5_NAME = DPT +OBS_VAR5_LEVELS = P700 +OBS_VAR5_THRESH = >=263, >=286, >=273 + +FCST_VAR6_NAME = WIND_P850_ENS_MEAN +FCST_VAR6_LEVELS = P850 +FCST_VAR6_THRESH = >=5, >=10, >=15 + +OBS_VAR6_NAME = WIND +OBS_VAR6_LEVELS = P850 +OBS_VAR6_THRESH = >=5, >=10, >=15 + +FCST_VAR7_NAME = WIND_P700_ENS_MEAN +FCST_VAR7_LEVELS = P700 +FCST_VAR7_THRESH = >=10, >=15, >=20 + +OBS_VAR7_NAME = WIND +OBS_VAR7_LEVELS = P700 +OBS_VAR7_THRESH = >=10, >=15, >=20 + +FCST_VAR8_NAME = WIND_P500_ENS_MEAN +FCST_VAR8_LEVELS = P500 +FCST_VAR8_THRESH = >=15, >=21, >=26 + +OBS_VAR8_NAME = WIND +OBS_VAR8_LEVELS = P500 +OBS_VAR8_THRESH = >=15, >=21, >=26 + +FCST_VAR9_NAME = WIND_P250_ENS_MEAN +FCST_VAR9_LEVELS = P250 +FCST_VAR9_THRESH = >=26, >=31, >=46, >=62 + +OBS_VAR9_NAME = WIND +OBS_VAR9_LEVELS = P250 +OBS_VAR9_THRESH = >=26, >=31, >=46, >=62 + +FCST_VAR10_NAME = HGT_P500_ENS_MEAN +FCST_VAR10_LEVELS = P500 +FCST_VAR10_THRESH = >=5400, >=5600, >=5880 + +OBS_VAR10_NAME = HGT +OBS_VAR10_LEVELS = P500 +OBS_VAR10_THRESH = >=5400, >=5600, >=5880 + +FCST_VAR11_NAME = CAPE_L0_ENS_MEAN +FCST_VAR11_LEVELS = L0 +FCST_VAR11_OPTIONS = cnt_thresh = [ >0 ]; +FCST_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 + +OBS_VAR11_NAME = CAPE +OBS_VAR11_LEVELS = L0-100000 +OBS_VAR11_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR11_THRESH = <=1000, >1000&&<2500, >2500&&<4000, >2500 + +FCST_VAR12_NAME = HPBL_Z0_ENS_MEAN +FCST_VAR12_LEVELS = Z0 +FCST_VAR12_THRESH = <500, <1500, >1500 + +OBS_VAR12_NAME = PBL +OBS_VAR12_LEVELS = L0 +OBS_VAR12_OPTIONS = desc = "TKE"; +OBS_VAR12_THRESH = <500, <1500, >1500 diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf index bb3790689d..cb653746b1 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -151,45 +151,300 @@ OBS_VAR3_NAME = TMP OBS_VAR3_LEVELS = P850 OBS_VAR3_THRESH = >=298 -FCST_VAR4_NAME = DPT_P850_ENS_FREQ_ge283 +FCST_VAR4_NAME = TMP_P700_ENS_FREQ_ge273 FCST_VAR4_LEVELS = (*,*) FCST_VAR4_THRESH = ==0.1 -OBS_VAR4_NAME = DPT -OBS_VAR4_LEVELS = P850 -OBS_VAR4_THRESH = >=283 +OBS_VAR4_NAME = TMP +OBS_VAR4_LEVELS = P700 +OBS_VAR4_THRESH = >=273 -FCST_VAR5_NAME = DPT_P850_ENS_FREQ_ge288 +FCST_VAR5_NAME = TMP_P700_ENS_FREQ_ge278 FCST_VAR5_LEVELS = (*,*) FCST_VAR5_THRESH = ==0.1 -OBS_VAR5_NAME = DPT -OBS_VAR5_LEVELS = P850 -OBS_VAR5_THRESH = >=288 +OBS_VAR5_NAME = TMP +OBS_VAR5_LEVELS = P700 +OBS_VAR5_THRESH = >=278 -FCST_VAR6_NAME = DPT_P850_ENS_FREQ_ge293 +FCST_VAR6_NAME = TMP_P700_ENS_FREQ_ge283 FCST_VAR6_LEVELS = (*,*) FCST_VAR6_THRESH = ==0.1 -OBS_VAR6_NAME = DPT -OBS_VAR6_LEVELS = P850 -OBS_VAR6_THRESH = >=293 +OBS_VAR6_NAME = TMP +OBS_VAR6_LEVELS = P700 +OBS_VAR6_THRESH = >=283 -FCST_VAR7_NAME = WIND_P850_ENS_FREQ_ge5 +FCST_VAR7_NAME = TMP_P500_ENS_FREQ_ge258 FCST_VAR7_LEVELS = (*,*) FCST_VAR7_THRESH = ==0.1 -OBS_VAR7_NAME = WIND -OBS_VAR7_LEVELS = P850 -OBS_VAR7_THRESH = >=5 +OBS_VAR7_NAME = TMP +OBS_VAR7_LEVELS = P500 +OBS_VAR7_THRESH = >=258 -FCST_VAR8_NAME = WIND_P850_ENS_FREQ_ge10 +FCST_VAR8_NAME = TMP_P500_ENS_FREQ_ge263 FCST_VAR8_LEVELS = (*,*) FCST_VAR8_THRESH = ==0.1 -OBS_VAR8_NAME = WIND -OBS_VAR8_LEVELS = P850 -OBS_VAR8_THRESH = >=10 +OBS_VAR8_NAME = TMP +OBS_VAR8_LEVELS = P500 +OBS_VAR8_THRESH = >=263 + +FCST_VAR9_NAME = TMP_P500_ENS_FREQ_ge268 +FCST_VAR9_LEVELS = (*,*) +FCST_VAR9_THRESH = ==0.1 + +OBS_VAR9_NAME = TMP +OBS_VAR9_LEVELS = P500 +OBS_VAR9_THRESH = >=268 + +FCST_VAR10_NAME = DPT_P850_ENS_FREQ_ge273 +FCST_VAR10_LEVELS = (*,*) +FCST_VAR10_THRESH = ==0.1 + +OBS_VAR10_NAME = DPT +OBS_VAR10_LEVELS = P850 +OBS_VAR10_THRESH = >=273 + +FCST_VAR11_NAME = DPT_P850_ENS_FREQ_ge278 +FCST_VAR11_LEVELS = (*,*) +FCST_VAR11_THRESH = ==0.1 + +OBS_VAR11_NAME = DPT +OBS_VAR11_LEVELS = P850 +OBS_VAR11_THRESH = >=278 + +FCST_VAR12_NAME = DPT_P850_ENS_FREQ_ge283 +FCST_VAR12_LEVELS = (*,*) +FCST_VAR12_THRESH = ==0.1 + +OBS_VAR12_NAME = DPT +OBS_VAR12_LEVELS = P850 +OBS_VAR12_THRESH = >=283 + +FCST_VAR13_NAME = DPT_P700_ENS_FREQ_ge263 +FCST_VAR13_LEVELS = (*,*) +FCST_VAR13_THRESH = ==0.1 + +OBS_VAR13_NAME = DPT +OBS_VAR13_LEVELS = P700 +OBS_VAR13_THRESH = >=263 + +FCST_VAR14_NAME = DPT_P700_ENS_FREQ_ge268 +FCST_VAR14_LEVELS = (*,*) +FCST_VAR14_THRESH = ==0.1 + +OBS_VAR14_NAME = DPT +OBS_VAR14_LEVELS = P700 +OBS_VAR14_THRESH = >=268 + +FCST_VAR15_NAME = DPT_P700_ENS_FREQ_ge273 +FCST_VAR15_LEVELS = (*,*) +FCST_VAR15_THRESH = ==0.1 + +OBS_VAR15_NAME = DPT +OBS_VAR15_LEVELS = P700 +OBS_VAR15_THRESH = >=273 + +FCST_VAR16_NAME = WIND_P850_ENS_FREQ_ge5 +FCST_VAR16_LEVELS = (*,*) +FCST_VAR16_THRESH = ==0.1 + +OBS_VAR16_NAME = WIND +OBS_VAR16_LEVELS = P850 +OBS_VAR16_THRESH = >=5 + +FCST_VAR17_NAME = WIND_P850_ENS_FREQ_ge10 +FCST_VAR17_LEVELS = (*,*) +FCST_VAR17_THRESH = ==0.1 + +OBS_VAR17_NAME = WIND +OBS_VAR17_LEVELS = P850 +OBS_VAR17_THRESH = >=10 + +FCST_VAR18_NAME = WIND_P850_ENS_FREQ_ge15 +FCST_VAR18_LEVELS = (*,*) +FCST_VAR18_THRESH = ==0.1 + +OBS_VAR18_NAME = WIND +OBS_VAR18_LEVELS = P850 +OBS_VAR18_THRESH = >=15 + +FCST_VAR19_NAME = WIND_P700_ENS_FREQ_ge10 +FCST_VAR19_LEVELS = (*,*) +FCST_VAR19_THRESH = ==0.1 + +OBS_VAR19_NAME = WIND +OBS_VAR19_LEVELS = P700 +OBS_VAR19_THRESH = >=10 + +FCST_VAR20_NAME = WIND_P700_ENS_FREQ_ge15 +FCST_VAR20_LEVELS = (*,*) +FCST_VAR20_THRESH = ==0.1 + +OBS_VAR20_NAME = WIND +OBS_VAR20_LEVELS = P700 +OBS_VAR20_THRESH = >=15 + +FCST_VAR21_NAME = WIND_P700_ENS_FREQ_ge20 +FCST_VAR21_LEVELS = (*,*) +FCST_VAR21_THRESH = ==0.1 + +OBS_VAR21_NAME = WIND +OBS_VAR21_LEVELS = P700 +OBS_VAR21_THRESH = >=20 + +FCST_VAR22_NAME = WIND_P500_ENS_FREQ_ge15 +FCST_VAR22_LEVELS = (*,*) +FCST_VAR22_THRESH = ==0.1 + +OBS_VAR22_NAME = WIND +OBS_VAR22_LEVELS = P500 +OBS_VAR22_THRESH = >=15 + +FCST_VAR23_NAME = WIND_P500_ENS_FREQ_ge21 +FCST_VAR23_LEVELS = (*,*) +FCST_VAR23_THRESH = ==0.1 + +OBS_VAR23_NAME = WIND +OBS_VAR23_LEVELS = P500 +OBS_VAR23_THRESH = >=21 + +FCST_VAR24_NAME = WIND_P500_ENS_FREQ_ge26 +FCST_VAR24_LEVELS = (*,*) +FCST_VAR24_THRESH = ==0.1 + +OBS_VAR24_NAME = WIND +OBS_VAR24_LEVELS = P500 +OBS_VAR24_THRESH = >=26 + +FCST_VAR25_NAME = WIND_P250_ENS_FREQ_ge26 +FCST_VAR25_LEVELS = (*,*) +FCST_VAR25_THRESH = ==0.1 + +OBS_VAR25_NAME = WIND +OBS_VAR25_LEVELS = P250 +OBS_VAR25_THRESH = >=26 + +FCST_VAR26_NAME = WIND_P250_ENS_FREQ_ge31 +FCST_VAR26_LEVELS = (*,*) +FCST_VAR26_THRESH = ==0.1 + +OBS_VAR26_NAME = WIND +OBS_VAR26_LEVELS = P250 +OBS_VAR26_THRESH = >=31 + +FCST_VAR27_NAME = WIND_P250_ENS_FREQ_ge36 +FCST_VAR27_LEVELS = (*,*) +FCST_VAR27_THRESH = ==0.1 + +OBS_VAR27_NAME = WIND +OBS_VAR27_LEVELS = P250 +OBS_VAR27_THRESH = >=36 + +FCST_VAR28_NAME = WIND_P250_ENS_FREQ_ge46 +FCST_VAR28_LEVELS = (*,*) +FCST_VAR28_THRESH = ==0.1 + +OBS_VAR28_NAME = WIND +OBS_VAR28_LEVELS = P250 +OBS_VAR28_THRESH = >=46 + +FCST_VAR29_NAME = WIND_P250_ENS_FREQ_ge62 +FCST_VAR29_LEVELS = (*,*) +FCST_VAR29_THRESH = ==0.1 + +OBS_VAR29_NAME = WIND +OBS_VAR29_LEVELS = P250 +OBS_VAR29_THRESH = >=62 + +FCST_VAR30_NAME = HGT_P500_ENS_FREQ_ge5400 +FCST_VAR30_LEVELS = (*,*) +FCST_VAR30_THRESH = ==0.1 + +OBS_VAR30_NAME = HGT +OBS_VAR30_LEVELS = P500 +OBS_VAR30_THRESH = >=5400 + +FCST_VAR31_NAME = HGT_P500_ENS_FREQ_ge5600 +FCST_VAR31_LEVELS = (*,*) +FCST_VAR31_THRESH = ==0.1 + +OBS_VAR31_NAME = HGT +OBS_VAR31_LEVELS = P500 +OBS_VAR31_THRESH = >=5600 + +FCST_VAR32_NAME = HGT_P500_ENS_FREQ_ge5880 +FCST_VAR32_LEVELS = (*,*) +FCST_VAR32_THRESH = ==0.1 + +OBS_VAR32_NAME = HGT +OBS_VAR32_LEVELS = P500 +OBS_VAR32_THRESH = >=5880 + +FCST_VAR33_NAME = CAPE_L0_ENS_FREQ_le1000 +FCST_VAR33_LEVELS = (*,*) +FCST_VAR33_THRESH = ==0.1 + +OBS_VAR33_NAME = CAPE +OBS_VAR33_LEVELS = L0-100000 +OBS_VAR33_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR33_THRESH = <=1000 + +FCST_VAR34_NAME = CAPE_L0_ENS_FREQ_gt1000.and.lt2500 +FCST_VAR34_LEVELS = (*,*) +FCST_VAR34_THRESH = ==0.1 + +OBS_VAR34_NAME = CAPE +OBS_VAR34_LEVELS = L0-100000 +OBS_VAR34_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR34_THRESH = >1000&&<2500 + +FCST_VAR35_NAME = CAPE_L0_ENS_FREQ_gt2500.and.lt4000 +FCST_VAR35_LEVELS = (*,*) +FCST_VAR35_THRESH = ==0.1 + +OBS_VAR35_NAME = CAPE +OBS_VAR35_LEVELS = L0-100000 +OBS_VAR35_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR35_THRESH = >2500&&<4000 + +FCST_VAR36_NAME = CAPE_L0_ENS_FREQ_gt2500 +FCST_VAR36_LEVELS = (*,*) +FCST_VAR36_THRESH = ==0.1 + +OBS_VAR36_NAME = CAPE +OBS_VAR36_LEVELS = L0-100000 +OBS_VAR36_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; +OBS_VAR36_THRESH =>2500 + +FCST_VAR37_NAME = HPBL_Z0_ENS_FREQ_lt500 +FCST_VAR37_LEVELS = (*,*) +FCST_VAR37_THRESH = ==0.1 + +OBS_VAR37_NAME = PBL +OBS_VAR37_LEVELS = L0 +OBS_VAR37_OPTIONS = desc = "TKE"; +OBS_VAR37_THRESH = <500 + +FCST_VAR38_NAME = HPBL_Z0_ENS_FREQ_lt1500 +FCST_VAR38_LEVELS = (*,*) +FCST_VAR38_THRESH = ==0.1 + +OBS_VAR38_NAME = PBL +OBS_VAR38_LEVELS = L0 +OBS_VAR38_OPTIONS = desc = "TKE"; +OBS_VAR38_THRESH = <1500 + +FCST_VAR39_NAME = HPBL_Z0_ENS_FREQ_gt1500 +FCST_VAR39_LEVELS = (*,*) +FCST_VAR39_THRESH = ==0.1 + +OBS_VAR39_NAME = PBL +OBS_VAR39_LEVELS = L0 +OBS_VAR39_OPTIONS = desc = "TKE"; +OBS_VAR39_THRESH = >1500 # Forecast data description variables FCST_IS_PROB = True From 01f1c53b0a24b1e4f0a73826691b57ad34e5ab0e Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 4 Jan 2022 06:28:40 -0500 Subject: [PATCH 109/203] Modify -N setting on Cray (#655) --- scripts/exregional_run_fcst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 8aa05ce34d..5963fd6bcf 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -108,7 +108,7 @@ case "$MACHINE" in if [ ${PE_MEMBER01} -gt 24 ];then APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" else - APRUN="aprun -b -j1 -n24 -N24 -d1 -cc depth" + APRUN="aprun -b -j1 -n${PE_MEMBER01} -N${PE_MEMBER01} -d1 -cc depth" fi ;; From c810e61810e3277273ecea986514fec31610f587 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 5 Jan 2022 06:07:48 -0500 Subject: [PATCH 110/203] Add a flag for MERRA2 (#659) --- scripts/exregional_run_fcst.sh | 28 ++++++++++--------- ...3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh | 2 ++ ush/config_defaults.sh | 9 ++++++ ush/generate_FV3LAM_wflow.sh | 12 ++++---- ush/setup.sh | 24 ++++++++++++++++ ush/valid_param_vals.sh | 1 + 6 files changed, 58 insertions(+), 18 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5963fd6bcf..513aa66345 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -410,20 +410,22 @@ done # #----------------------------------------------------------------------- # -for f_nm_path in ${FIXclim}/*; do - f_nm=$( basename "${f_nm_path}" ) - pre_f="${f_nm%%.*}" - - if [ "${pre_f}" = "merra2" ]; then - mnth=$( printf "%s\n" "${f_nm}" | grep -o -P '(?<=2014.m).*(?=.nc)' ) - symlink="${run_dir}/aeroclim.m${mnth}.nc" - else - symlink="${run_dir}/${pre_f}.dat" - fi - target="${f_nm_path}" - create_symlink_to_file target="$target" symlink="$symlink" \ +if [ "${USE_MERRA_CLIMO}" = "TRUE" ]; then + for f_nm_path in ${FIXclim}/*; do + f_nm=$( basename "${f_nm_path}" ) + pre_f="${f_nm%%.*}" + + if [ "${pre_f}" = "merra2" ]; then + mnth=$( printf "%s\n" "${f_nm}" | grep -o -P '(?<=2014.m).*(?=.nc)' ) + symlink="${run_dir}/aeroclim.m${mnth}.nc" + else + symlink="${run_dir}/${pre_f}.dat" + fi + target="${f_nm_path}" + create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -done + done +fi # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh index f391b26886..03dbd2c381 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh @@ -13,6 +13,8 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" CCPP_PHYS_SUITE="FV3_GFS_v15_thompson_mynn_lam3km" +USE_MERRA_CLIMO="TRUE" + EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" USE_USER_STAGED_EXTRN_FILES="TRUE" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 702cf4db8d..c3090cc8bc 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1333,6 +1333,15 @@ RUN_TASK_VX_ENSPOINT="FALSE" # #----------------------------------------------------------------------- # +# Flag that determines whether MERRA2 aerosol climatology data and +# lookup tables for optics properties are obtained +# +#----------------------------------------------------------------------- +# +USE_MERRA_CLIMO="FALSE" +# +#----------------------------------------------------------------------- +# # Set the array parameter containing the names of all the fields that the # MAKE_SFC_CLIMO_TN task generates on the native FV3-LAM grid. # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index d19a58b52c..db65baec21 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -618,18 +618,20 @@ fi # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " +if [ "${USE_MERRA_CLIMO}" = "TRUE" ]; then + print_info_msg "$VERBOSE" " Copying MERRA2 aerosol climatology data files from system directory (FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: FIXaer = \"${FIXaer}\" FIXlut = \"${FIXlut}\" FIXclim = \"${FIXclim}\"" -check_for_preexist_dir_file "${FIXclim}" "delete" -mkdir_vrfy -p "${FIXclim}" + check_for_preexist_dir_file "${FIXclim}" "delete" + mkdir_vrfy -p "${FIXclim}" -cp_vrfy "${FIXaer}/merra2.aerclim"*".nc" "${FIXclim}/" -cp_vrfy "${FIXlut}/optics"*".dat" "${FIXclim}/" + cp_vrfy "${FIXaer}/merra2.aerclim"*".nc" "${FIXclim}/" + cp_vrfy "${FIXlut}/optics"*".dat" "${FIXclim}/" +fi # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index ee49750ae9..88dcab21e5 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -790,6 +790,30 @@ check_var_valid_value \ # #----------------------------------------------------------------------- # +# Make sure that USE_MERRA_CLIMO is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "USE_MERRA_CLIMO" "valid_vals_USE_MERRA_CLIMO" +# +# Set USE_MERRA_CLIMO to either "TRUE" or "FALSE" so we don't +# have to consider other valid values later on. +# +USE_MERRA_CLIMO=$(echo_uppercase $USE_MERRA_CLIMO) +if [ "${USE_MERRA_CLIMO}" = "TRUE" ] || \ + [ "${USE_MERRA_CLIMO}" = "YES" ]; then + USE_MERRA_CLIMO="TRUE" +elif [ "${USE_MERRA_CLIMO}" = "FALSE" ] || \ + [ "${USE_MERRA_CLIMO}" = "NO" ]; then + USE_MERRA_CLIMO="FALSE" +fi +# Force to "TRUE" in case of FV3_GFS_v15_thompson_mynn_lam3km: +if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15_thompson_mynn_lam3km" ]; then + USE_MERRA_CLIMO="TRUE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that FCST_MODEL is set to a valid value. # #----------------------------------------------------------------------- diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a8bd0ae721..c421441309 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -78,3 +78,4 @@ valid_vals_FVCOM_WCSTART=("warm" "WARM" "cold" "COLD") valid_vals_COMPILER=("intel" "gnu") valid_vals_SUB_HOURLY_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") +valid_vals_USE_MERRA_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From 6186978b36e13a354aa7f68072b6518a0d33146a Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Mon, 10 Jan 2022 15:02:11 -0700 Subject: [PATCH 111/203] Verification modifications to conf files. (#662) --- ush/templates/parm/metplus/PointStat_conus_sfc.conf | 8 ++++---- ush/templates/parm/metplus/PointStat_upper_air.conf | 4 ++-- ush/templates/parm/metplus/REFC.conf | 4 ++-- ush/templates/parm/metplus/RETOP.conf | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 4eccff4c18..779fa73b45 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -83,7 +83,7 @@ PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_WIND, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS # For defining the time periods for summarization # False for no time summary, True otherwise @@ -147,7 +147,7 @@ BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts BOTH_VAR6_NAME = WIND BOTH_VAR6_LEVELS = Z10 -BOTH_VAR6_THRESH = >=2.572 ;; m/s or 5kts +BOTH_VAR6_THRESH = >=2.572, >=2.572 && <5.144, >=5.144, >=10.288, >=15.433 ;; m/s or 5, 10, 20, 30kts BOTH_VAR6_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind BOTH_VAR7_NAME = PRMSL @@ -155,7 +155,7 @@ BOTH_VAR7_LEVELS = Z0 FCST_VAR8_NAME = TCDC FCST_VAR8_LEVELS = L0 -FCST_VAR8_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = 8; GRIB2_ipdtmpl_val = {lead?fmt=%H}; +BOTH_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR8_NAME = TCDC OBS_VAR8_LEVELS = L0 OBS_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } @@ -163,7 +163,7 @@ OBS_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } BOTH_VAR9_NAME = VIS BOTH_VAR9_LEVELS = L0 BOTH_VAR9_THRESH = <805, <1609, <4828, <8045 ,>=8045, <16090 -BOTH_VAR9_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } +BOTH_VAR9_OPTIONS = censor_thresh = [>16090]; censor_val = [16090]; interp = { type = [ { method = NEAREST; width = 1; } ]; } BOTH_VAR10_NAME = GUST BOTH_VAR10_LEVELS = Z0 diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index cd05d930b2..ca60b7d5d0 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -83,7 +83,7 @@ PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, D_WIND, HOVI, CEILING, D_PBL, D_CAPE, MXGS # For defining the time periods for summarization # False for no time summary, True otherwise @@ -147,7 +147,7 @@ BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts BOTH_VAR6_NAME = WIND BOTH_VAR6_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 -BOTH_VAR6_THRESH = >=2.572 ;; m/s or 5kts +BOTH_VAR6_THRESH = >=2.572, >=2.572 && <5.144, >=5.144, >=10.288, >=15.433, >=20.577, >=25.722 ;; m/s or 5, 10, 20, 30, 40, 50kts BOTH_VAR7_NAME = HGT BOTH_VAR7_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf index 9a3fdb2d1c..a661fabc89 100644 --- a/ush/templates/parm/metplus/REFC.conf +++ b/ush/templates/parm/metplus/REFC.conf @@ -82,13 +82,13 @@ GRID_STAT_REGRID_TO_GRID = FCST # Forecast/Observation variable Information FCST_VAR1_NAME = REFC FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; +FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 OBS_VAR1_NAME = MergedReflectivityQCComposite OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; +OBS_VAR1_OPTIONS = censor_thresh = [eq-999, <-20]; censor_val = [-9999, -20]; cnt_thresh = [ >15 ]; cnt_logic = UNION; OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 diff --git a/ush/templates/parm/metplus/RETOP.conf b/ush/templates/parm/metplus/RETOP.conf index 5508bd600c..c09ea7a18d 100644 --- a/ush/templates/parm/metplus/RETOP.conf +++ b/ush/templates/parm/metplus/RETOP.conf @@ -82,13 +82,13 @@ GRID_STAT_REGRID_TO_GRID = FCST # Forecast/Observation variable Information FCST_VAR1_NAME = RETOP FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; convert(x) = M_to_KFT(x); +FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); cnt_thresh = [ >0 ]; cnt_logic = UNION; BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 OBS_VAR1_NAME = EchoTop18 OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +OBS_VAR1_OPTIONS = convert(x) = KM_to_KFT(x); censor_thresh = [<=-9.84252,eq-3.28084]; censor_val = [-9999,-16.4042]; cnt_thresh = [ >0 ]; cnt_logic = UNION; OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 From ac662c738d5823566b0e33739a0d845e1ab1f2ea Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Wed, 12 Jan 2022 18:56:02 -0600 Subject: [PATCH 112/203] Refactor supported input path handling. (#653) ## DESCRIPTION OF CHANGES: This PR is one of two in a series that addresses refactoring platform-dependence. The topic of this PR is to refactor the logic for supporting platforms with specific real-time data feeds. Here is a list of features and modifications that were made: - Apply appropriate style for functions, mainly related to white space for nested code and comments. - Make external model file offset hours configurable for analysis- and forecast-type files because sometimes we like to start a forecast from a forecast of another model. - Refactor to reduce duplication of information. - Set up filenames and paths to be specified with templates in a consistent way for every model instead of with bash logic that is different for every model's naming convention. - Remove paths that do not exist on platforms I have access to: Jet and Hera. No changes were made to the logic for input managed by USE_USER_STAGED_EXTRN_FILES or COMINGgfs for NCO mode, although it could make sense to re-assess the NCO mode handling at a later date. I plan to go through and "review" the code to lead reviewers through this one since it bit of change. It may be helpful to view it using GitHub's ignore whitespace feature. ## TESTS CONDUCTED: Test cases using the WE2E test on Hera; see PR for full list. A test case for the same forecast configuration using known paths on Hera to exercise the new code. I checked that arrays were consistent, that the script exits in a sane manner when files are not available on disk or HPSS, and that I haven't broken anything with the way files are handled through the "user specified" mechanism necessary for the test framework. ## ISSUE (optional): This work is an incremental change in support of Issue #618 ## CONTRIBUTORS (optional): @christopherwharrop-noaa @venitahagerty @robgonzalezpita --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 25 +- ush/config_defaults.sh | 20 + ush/get_extrn_mdl_file_dir_info.sh | 1023 ++++++++++------------------ ush/set_extrn_mdl_params.sh | 384 +++-------- 4 files changed, 477 insertions(+), 975 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index bfe2d7bcea..a452062c5f 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -167,28 +167,29 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" # #----------------------------------------------------------------------- # -# Set the parameter anl_or_fcst that determines whether we want to get -# analysis or forecast files. This depends on whether we want these files -# to generate initial condition and surface field files or lateral boundary -# condition files. Also, set time_offset_hrs, which is the offset in -# hours between the current cycle's starting time and the starting time -# of the external model providing the LBCs. +# Set parameters for grabbing either the initial conditions from analysis or +# forecast files of external models, or the lateral boundary conditions +# from external models. The script has been called to do the work for +# one or the other. # #----------------------------------------------------------------------- # if [ "${ICS_OR_LBCS}" = "ICS" ]; then - anl_or_fcst="ANL" - time_offset_hrs="0" + if [ ${EXTRN_MDL_ICS_OFFSET_HRS} -eq 0 ] ; then + anl_or_fcst="ANL" + time_offset_hrs=0 + else + anl_or_fcst="FCST" + time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} + fi elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then anl_or_fcst="FCST" - time_offset_hrs="${EXTRN_MDL_LBCS_OFFSET_HRS}" + time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} fi # #----------------------------------------------------------------------- # -# Set the name of and then create the directory in which to stage the -# external model files for the current cycle (if it doesn't already exist). -# Then change location to that directory. +# Create the directory where the exetrnal model files should be stored # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index c3090cc8bc..21d6096a16 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -604,6 +604,24 @@ NDAS_OBS_DIR="/path/to/observation-directory/ndas/proc" # data availble at least every 6 hours. It is up to the user to ensure # that this is the case. # +# EXTRN_MDL_ICS_OFFSET_HRS: +# Users may wish to start a forecast from a forecast of a previous cycle +# of an external model. This variable sets the number of hours earlier +# the external model started than when the FV3 forecast configured here +# should start. For example, the forecast should start from a 6 hour +# forecast of the GFS, then EXTRN_MDL_ICS_OFFSET_HRS=6. + +# EXTRN_MDL_LBCS_OFFSET_HRS: +# Users may wish to use lateral boundary conditions from a forecast that +# was started earlier than the initial time for the FV3 forecast +# configured here. This variable sets the number of hours earlier +# the external model started than when the FV3 forecast configured here +# should start. For example, the forecast should use lateral boundary +# conditions from the GFS started 6 hours earlier, then +# EXTRN_MDL_LBCS_OFFSET_HRS=6. +# Note: the default value is model-dependent and set in +# set_extrn_mdl_params.sh +# # FV3GFS_FILE_FMT_ICS: # If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS # is set to "FV3GFS"), this variable specifies the format of the model @@ -619,6 +637,8 @@ NDAS_OBS_DIR="/path/to/observation-directory/ndas/proc" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" LBC_SPEC_INTVL_HRS="6" +EXTRN_MDL_ICS_OFFSET_HRS="0" +EXTRN_MDL_LBCS_OFFSET_HRS="" FV3GFS_FILE_FMT_ICS="nemsio" FV3GFS_FILE_FMT_LBCS="nemsio" # diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index ff7d1bfda1..fdab7dad41 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -18,83 +18,10 @@ # #----------------------------------------------------------------------- # -function get_extrn_mdl_file_dir_info() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "extrn_mdl_name" \ - "anl_or_fcst" \ - "cdate_FV3LAM" \ - "time_offset_hrs" \ - "varname_extrn_mdl_cdate" \ - "varname_extrn_mdl_lbc_spec_fhrs" \ - "varname_extrn_mdl_fns_on_disk" \ - "varname_extrn_mdl_fns_in_arcv" \ - "varname_extrn_mdl_sysdir" \ - "varname_extrn_mdl_arcv_fmt" \ - "varname_extrn_mdl_arcv_fns" \ - "varname_extrn_mdl_arcv_fps" \ - "varname_extrn_mdl_arcvrel_dir" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script/function. Note that these will be printed out only if VERBOSE -# is set to TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# -if [ 0 = 1 ]; then - if [ "$#" -ne "13" ]; then +usage () { - print_err_msg_exit " +echo " Incorrect number of arguments specified: Function name: \"${func_name}\" @@ -117,7 +44,7 @@ Usage: varname_extrn_mdl_arcvrel_dir where the arguments are defined as follows: - + extrn_mdl_name: Name of the external model, i.e. the name of the model providing the fields from which files containing initial conditions, surface fields, @@ -160,9 +87,13 @@ where the arguments are defined as follows: boundary condition (LBC) output files are obtained from the external model (and will be used to update the LBCs of the FV3-LAM). - varname_extrn_mdl_fns: - Name of the global variable that will contain the names of the exter- - nal model output files. + varname_extrn_mdl_fns_on_disk: + Name of the global variable that will contain the expected names of + the external model output files on disk. + + varname_extrn_mdl_fns_in_arcv: + Name of the global variable that will contain the expected names of + the external model output files on NOAA HPSS. varname_extrn_mdl_sysdir: Name of the global variable that will contain the system directory in @@ -187,32 +118,8 @@ where the arguments are defined as follows: rectory, i.e. the directory \"inside\" the archive file in which the ex- ternal model output files may be stored. " +} - fi - -fi - - -# -#----------------------------------------------------------------------- -# -# Declare additional local variables. -# -#----------------------------------------------------------------------- -# - local yyyy mm dd hh mn yyyymmdd \ - lbc_spec_fhrs i num_fhrs \ - yy ddd fcst_hhh fcst_hh fcst_mn \ - prefix suffix fns fns_on_disk fns_in_arcv \ - sysbasedir sysdir \ - arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir -# -#----------------------------------------------------------------------- -# -# Declare local function to avoid repetition -# -#----------------------------------------------------------------------- -# function quit_unless_user_spec_data() { if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then print_err_msg_exit "\ @@ -222,236 +129,195 @@ has not been specified for this external model and machine combination: MACHINE = \"$MACHINE\"" fi } -# -#----------------------------------------------------------------------- -# -# Check input variables for valid values. -# -#----------------------------------------------------------------------- -# + +function get_extrn_mdl_file_dir_info() { + + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + + local func_name="${FUNCNAME[0]}" + # + #----------------------------------------------------------------------- + # + # Specify the set of valid argument names for this script/function. Then + # process the arguments provided to this script/function (which should + # consist of a set of name-value pairs of the form arg1="value1", etc). + # + #----------------------------------------------------------------------- + # + local valid_args=( \ + "extrn_mdl_name" \ + "anl_or_fcst" \ + "cdate_FV3LAM" \ + "time_offset_hrs" \ + "varname_extrn_mdl_cdate" \ + "varname_extrn_mdl_lbc_spec_fhrs" \ + "varname_extrn_mdl_fns_on_disk" \ + "varname_extrn_mdl_fns_in_arcv" \ + "varname_extrn_mdl_sysdir" \ + "varname_extrn_mdl_arcv_fmt" \ + "varname_extrn_mdl_arcv_fns" \ + "varname_extrn_mdl_arcv_fps" \ + "varname_extrn_mdl_arcvrel_dir" \ + ) + process_args valid_args "$@" + + if [ "$#" -ne "13" ]; then + print_err_msg_exit $(usage) + fi + + # + #----------------------------------------------------------------------- + # + # For debugging purposes, print out values of arguments passed to this + # script/function. Note that these will be printed out only if VERBOSE + # is set to TRUE. + # + #----------------------------------------------------------------------- + # + print_input_args valid_args + + # + #----------------------------------------------------------------------- + # + # Declare additional local variables. + # + #----------------------------------------------------------------------- + # + local yyyy yy mm dd hh mn yyyymmdd ddd \ + lbc_spec_fhrs i num_fhrs \ + fcst_hhh fcst_hh fcst_mn \ + prefix suffix fns fns_on_disk fns_in_arcv \ + sysbasedir sysdir \ + arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir + anl_or_fcst=$(echo_uppercase $anl_or_fcst) valid_vals_anl_or_fcst=( "ANL" "FCST" ) check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" -# -#----------------------------------------------------------------------- -# -# Extract from cdate_FV3LAM the starting year, month, day, and hour of -# the FV3-LAM cycle. Then subtract the temporal offset specified in -# time_offset_hrs (assumed to be given in units of hours) from cdate_FV3LAM -# to obtain the starting date and time of the external model, express the -# result in YYYYMMDDHH format, and save it in cdate. This is the starting -# time of the external model forecast. -# -#----------------------------------------------------------------------- -# - yyyy=${cdate_FV3LAM:0:4} - mm=${cdate_FV3LAM:4:2} - dd=${cdate_FV3LAM:6:2} + # + #----------------------------------------------------------------------- + # + # Set cdate to the start time for the external model being used. + # + #----------------------------------------------------------------------- + # hh=${cdate_FV3LAM:8:2} yyyymmdd=${cdate_FV3LAM:0:8} + # Adjust time for offset cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) -# -#----------------------------------------------------------------------- -# -# Extract from cdate the starting year, month, day, and hour of the external -# model forecast. Also, set the starting minute to "00" and get the date -# without the time-of-day. These are needed below in setting various -# directory and file names. -# -#----------------------------------------------------------------------- -# + yyyy=${cdate:0:4} + yy=${yyyy:2:4} mm=${cdate:4:2} dd=${cdate:6:2} hh=${cdate:8:2} mn="00" yyyymmdd=${cdate:0:8} -# -#----------------------------------------------------------------------- -# -# Initialize lbc_spec_fhrs to an empty array. Then, if considering a -# forecast, reset lbc_spec_fhrs to the array of forecast hours at which -# the lateral boundary conditions (LBCs) are to be updated, starting with -# the 2nd such time (i.e. the one having array index 1). We do not include -# the first hour (hour 0) because at this initial time, the LBCs are -# obtained from the analysis fields provided by the external model (as -# opposed to a forecast field). -# -#----------------------------------------------------------------------- -# + # Julian day -- not 3 digit day of month + ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) + # + #----------------------------------------------------------------------- + # + # Initialize lbc_spec_fhrs array. Skip the initial time, since it is + # handled separately. + # + #----------------------------------------------------------------------- + # lbc_spec_fhrs=( "" ) if [ "${anl_or_fcst}" = "FCST" ]; then lbc_spec_fhrs=( "${LBC_SPEC_FCST_HRS[@]}" ) -# -# Add the temporal offset specified in time_offset_hrs (assumed to be in -# units of hours) to the the array of LBC update forecast hours to make -# up for shifting the starting hour back in time. After this addition, -# lbc_spec_fhrs will contain the LBC update forecast hours relative to -# the start time of the external model run. -# + num_fhrs=${#lbc_spec_fhrs[@]} for (( i=0; i<=$((num_fhrs-1)); i++ )); do + # Add in offset to account for shift in initial time lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + time_offset_hrs )) done fi -# -#----------------------------------------------------------------------- -# -# Set additional parameters needed in forming the names of the external -# model files only under certain circumstances. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_name}" = "RAP" ] || \ - [ "${extrn_mdl_name}" = "HRRR" ] || \ - [ "${extrn_mdl_name}" = "NAM" ] || \ - [ "${extrn_mdl_name}" = "FV3GFS" -a "${MACHINE}" = "JET" ]; then -# -# Get the Julian day-of-year of the starting date and time of the exter- -# nal model forecast. -# - ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) -# -# Get the last two digits of the year of the starting date and time of -# the external model forecast. -# - yy=${yyyy:2:4} + # + #----------------------------------------------------------------------- + # + # The model may be started with a variety of file types from FV3GFS. + # Set that file type now + # + #----------------------------------------------------------------------- + # - fi -# -#----------------------------------------------------------------------- -# -# Set the external model output file names that must be obtained (from -# disk if available, otherwise from HPSS). -# -#----------------------------------------------------------------------- -# if [ "${anl_or_fcst}" = "ANL" ]; then fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" elif [ "${anl_or_fcst}" = "FCST" ]; then fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" fi + # + #----------------------------------------------------------------------- + # + # Generate an array of file names expected from the external model + # Assume that filenames in archive and on disk are the same, unless + # otherwise specified (primarily on Jet). + # + #----------------------------------------------------------------------- + # + declare -a fns_on_disk + declare -a fns_in_arcv case "${anl_or_fcst}" in -# -#----------------------------------------------------------------------- -# -# Consider analysis files (possibly including surface files). -# -#----------------------------------------------------------------------- -# - "ANL") - - fcst_hh="00" - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") -# fns=( "atm" "sfc" "nst" ) - fns=( "atm" "sfc" ) - prefix="gfs.t${hh}z." - fns=( "${fns[@]/#/$prefix}" ) - suffix="anl.nemsio" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - fns=( "atm" "sfc" ) - suffix="anl.nemsio" - fns=( "${fns[@]/%/$suffix}" ) - -# Set names of external files if searching on disk. - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." - else - prefix="gfs.t${hh}z." + "ANL") + + fcst_hh="00" + fcst_mn="00" + + case "${extrn_mdl_name}" in + + "GSMGFS") + fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") + ;; + + "FV3GFS") + case "${fv3gfs_file_fmt}" in + "nemsio") + fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") + + # File names are prefixed with a date time on Jet + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00" + fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) + fi + ;; + "grib2") + fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) + ;; + "netcdf") + fns_in_arcv=("gfs.t${hh}z.atmanl.nc" "gfs.t${hh}z.sfcanl.nc") + # File names are prefixed with a date time on Jet + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00" + fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) + fi + ;; + esac + ;; + + "RAP") + ;& # Fall through. RAP and HRRR follow same naming rules + + "HRRR") + fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) + if [ "${MACHINE}" = "JET" ]; then + fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_mn}${fcst_hh}${fcst_mn}" ) fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - -# Set names of external files if searching in an archive file, e.g. from -# HPSS. - prefix="gfs.t${hh}z." - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - -# GSK 12/16/2019: -# Turns out that the .f000 file contains certain necessary fields that -# are not in the .anl file, so switch to the former. -# fns=( "gfs.t${hh}z.pgrb2.0p25.anl" ) # Get only 0.25 degree files for now. -# fns=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - fns_on_disk=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + ;; - fns=( "atm" "sfc" ) - suffix="anl.nc" - fns=( "${fns[@]/%/$suffix}" ) + "NAM") + fns=( "" ) + fns_in_arcv=( "nam.t${hh}z.bgrdsf${fcst_hh}.tm00" ) + ;; -# Set names of external files if searching on disk. - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." - else - prefix="gfs.t${hh}z." - fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - -# Set names of external files if searching in an archive file, e.g. from -# HPSS. - prefix="gfs.t${hh}z." - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - fi - ;; - - "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "wrfnat_130_${fcst_hh}.grib2" ) - else - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - fi - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - ;; - - "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "wrfnat_hrconus_${fcst_hh}.grib2" ) - else - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - fi - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - ;; - - "NAM") - fns=( "" ) - prefix="nam.t${hh}z.bgrdsfi${hh}" - fns=( "${fns[@]/#/$prefix}" ) - suffix=".tm${hh}" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ + *) + if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then + print_err_msg_exit "\ The external model file names (either on disk or in archive files) have not yet been specified for this combination of external model (extrn_mdl_name) and analysis or forecast (anl_or_fcst): @@ -460,343 +326,182 @@ and analysis or forecast (anl_or_fcst): fi ;; - esac + esac # End external model case for ANL files ;; -# -#----------------------------------------------------------------------- -# -# Consider forecast files. -# -#----------------------------------------------------------------------- -# - "FCST") - - fcst_mn="00" - - case "${extrn_mdl_name}" in - "GSMGFS") + "FCST") + fcst_mn="00" fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="gfs.t${hh}z.atmf" - fns=( "${fcst_hhh[@]/#/$prefix}" ) - suffix=".nemsio" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - suffix=".nemsio" - fns=( "${fcst_hhh[@]/%/$suffix}" ) + fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" - else - prefix="gfs.t${hh}z.atmf" + case "${extrn_mdl_name}" in + + "GSMGFS") + fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" + ;; + + "FV3GFS") + + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then + fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" + if [ "${MACHINE}" = "JET" ]; then + disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nemsio" + for fhr in ${fcst_hhh[@]} ; do + fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) + done + fi + elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then + fn_tmpl="gfs.t${hh}z.pgrb2.0p25.fFHR3" + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + fn_tmpl="gfs.t${hh}z.atmfFHR3.nc" + if [ "${MACHINE}" = "JET" ]; then + disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nc" + for fhr in ${fcst_hhh[@]} ; do + fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) + done + fi fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - - prefix="gfs.t${hh}z.atmf" - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="gfs.t${hh}z.pgrb2.0p25.f" - fns_on_disk=( "${fcst_hhh[@]/#/$prefix}" ) - fns_in_arcv=( "${fcst_hhh[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - suffix=".nc" - fns=( "${fcst_hhh[@]/%/$suffix}" ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" - else - prefix="gfs.t${hh}z.atmf" + ;; + + "RAP") + ;& # Fall through since RAP and HRRR are named the same + + "HRRR") + fn_tmpl="${yy}${ddd}${hh}00FHR200" + if [ "${MACHINE}" = "JET" ]; then + disk_tmpl="${yy}${ddd}${hh}0000FHR2" + for fhr in ${fcst_hhh[@]} ; do + fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) + done fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - - prefix="gfs.t${hh}z.atmf" - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - fi - ;; - - "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="wrfnat_130_" - suffix=".grib2" - else - prefix="${yy}${ddd}${hh}${mn}" - suffix="${fcst_mn}" - fi - fns_on_disk=( "${fcst_hh[@]/#/$prefix}" ) - fns_on_disk=( "${fns_on_disk[@]/%/$suffix}" ) - - prefix="${yy}${ddd}${hh}${mn}" - fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) - suffix="${fcst_mn}" - fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) - ;; - - "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="wrfnat_hrconus_" - suffix=".grib2" - else - prefix="${yy}${ddd}${hh}${mn}" - suffix="${fcst_mn}" - fi - fns_on_disk=( "${fcst_hh[@]/#/$prefix}" ) - fns_on_disk=( "${fns_on_disk[@]/%/$suffix}" ) + ;; - prefix="${yy}${ddd}${hh}${mn}" - fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) - suffix="${fcst_mn}" - fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) - ;; + "NAM") + fn_tmpl="nam.t${hh}z.bgrdsfFHR3" + ;; - "NAM") - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="nam.t${hh}z.bgrdsf" - fns=( "${fcst_hhh[@]/#/$prefix}" ) - suffix="" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ + *) + if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then + print_err_msg_exit "\ The external model file names have not yet been specified for this com- bination of external model (extrn_mdl_name) and analysis or forecast (anl_or_fcst): extrn_mdl_name = \"${extrn_mdl_name}\" anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; + fi + ;; - esac + esac # End external model case for FCST files ;; + esac # End ANL FCST case + + # + # Expand the archive file names for all forecast hours + # + if [ ${anl_or_fcst} = FCST ] ; then + if [[ $fn_tmpl =~ FHR3 ]] ; then + fhrs=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) + tmpl=FHR3 + elif [[ ${fn_tmpl} =~ FHR2 ]] ; then + fhrs=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) + tmpl=FHR2 + else + print_err_msg_exit "\ + Forecast file name templates are expected to contain a template + string, either FHR2 or FHR3" + fi + for fhr in ${fhrs[@]}; do + fns_in_arcv+=(${fn_tmpl/$tmpl/$fhr}) + done + fi - esac -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. a directory on disk) in which the external -# model output files for the specified cycle date (cdate) may be located. -# Note that this will be used by the calling script only if the output -# files for the specified cdate actually exist at this location. Otherwise, -# the files will be searched for on the mass store (HPSS). -# -#----------------------------------------------------------------------- -# + # Make sure all filenames variables are set. + if [ -z $fns_in_arcv ] ; then + print_err_msg_exit "\ + The script has not set \$fns_in_arcv properly" + fi + + if [ -z ${fns_on_disk:-} ] ; then + fns_on_disk=(${fns_in_arcv[@]}) + fi + # + #----------------------------------------------------------------------- + # + # Set the system directory (i.e. a directory on disk) in which the external + # model output files for the specified cycle date (cdate) may be located. + # Note that this will be used by the calling script only if the output + # files for the specified cdate actually exist at this location. Otherwise, + # the files will be searched for on the mass store (HPSS). + # + #----------------------------------------------------------------------- + # if [ "${anl_or_fcst}" = "ANL" ]; then - sysbasedir="${EXTRN_MDL_SYSBASEDIR_ICS}" + sysbasedir=${EXTRN_MDL_SYSBASEDIR_ICS} elif [ "${anl_or_fcst}" = "FCST" ]; then - sysbasedir="${EXTRN_MDL_SYSBASEDIR_LBCS}" + sysbasedir=${EXTRN_MDL_SYSBASEDIR_LBCS} fi - sysdir="" - case "${extrn_mdl_name}" in - -# -# It is not clear which, if any, systems the (old) spectral GFS model is -# available on, so set sysdir for this external model to a null string. -# - "GSMGFS") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="" - ;; - "WCOSS_DELL_P3") - sysdir="" - ;; - "HERA") - sysdir="" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="" - ;; - "STAMPEDE") - sysdir="$sysbasedir" - ;; - *) - quit_unless_user_spec_data - ;; - esac - ;; - - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "HERA") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir" - ;; - "ODIN") - sysdir="$sysbasedir/${yyyymmdd}" - ;; - "CHEYENNE") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" - ;; - "STAMPEDE") - sysdir="$sysbasedir" - ;; - *) - quit_unless_user_spec_data - ;; - esac - ;; - - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - quit_unless_user_spec_data - ;; - esac - ;; + sysdir=$sysbasedir + # Use the basedir unless otherwise specified for special platform + # cases below. + if [ -n "${sysbasedir}" ] ; then + case "${extrn_mdl_name}" in + "FV3GFS") + case "$MACHINE" in + "WCOSS_CRAY") + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" + ;; + "WCOSS_DELL_P3") + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" + ;; + "HERA") + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" + ;; + "ODIN") + sysdir="$sysbasedir/${yyyymmdd}" + ;; + "CHEYENNE") + sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" + ;; + esac + ;; + + "RAP") + case "$MACHINE" in + "JET") + sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" + ;; + esac + ;; + + "HRRR") + case "$MACHINE" in + "JET") + sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" + ;; + esac + ;; - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - quit_unless_user_spec_data - ;; esac - ;; - - "NAM") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - quit_unless_user_spec_data - ;; - esac - ;; - - - *) - quit_unless_user_spec_data - esac -# -#----------------------------------------------------------------------- -# -# Set parameters associated with the mass store (HPSS) for the specified -# cycle date (cdate). These consist of: -# -# 1) The type of the archive file (e.g. tar, zip, etc). -# 2) The name of the archive file. -# 3) The full path in HPSS to the archive file. -# 4) The relative directory in the archive file in which the model output -# files are located. -# -# Note that these will be used by the calling script only if the archive -# file for the specified cdate actually exists on HPSS. -# -#----------------------------------------------------------------------- -# + fi + # + #----------------------------------------------------------------------- + # + # Set parameters associated with the mass store (HPSS) for the specified + # cycle date (cdate). These consist of: + # + # 1) The type of the archive file (e.g. tar, zip, etc). + # 2) The name of the archive file. + # 3) The full path in HPSS to the archive file. + # 4) The relative directory in the archive file in which the model output + # files are located. + # + # Note that these will be used by the calling script only if the archive + # file for the specified cdate actually exists on HPSS. + # + #----------------------------------------------------------------------- + # case "${extrn_mdl_name}" in "GSMGFS") @@ -890,30 +595,29 @@ bination of external model (extrn_mdl_name) and analysis or forecast "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# -# The zip archive files for RAPX are named such that the forecast files -# for odd-numbered starting hours (e.g. 01, 03, ..., 23) are stored -# together with the forecast files for the corresponding preceding even- -# numbered starting hours (e.g. 00, 02, ..., 22, respectively), in an -# archive file whose name contains only the even-numbered hour. Thus, -# in forming the name of the archive file, if the starting hour (hh) is -# odd, we reduce it by one to get the corresponding even-numbered hour -# and use that to form the archive file name. -# + # + # Note that this is GSL RAPX data, not operational NCEP RAP data. + # An option for the latter may be added in the future. + # + # The zip archive files for RAPX are named such that the forecast + # files for odd-numbered starting hours (e.g. 01, 03, ..., 23) are + # stored together with the forecast files for the corresponding + # preceding even numbered starting hours (e.g. 00, 02, ..., 22, + # respectively), in an archive file whose name contains only the + # even-numbered hour. Thus, in forming the name of the archive + # file, if the starting hour (hh) is odd, we reduce it by one to get + # the corresponding even-numbered hour and use that to form the + # archive file name. + # + # Convert hh to a decimal (i.e. base-10) number to ovoid octal + # interpretation in bash. + hh_orig=$hh -# Convert hh to a decimal (i.e. base-10) number. We need this because -# if it starts with a 0 (e.g. 00, 01, ..., 09), bash will treat it as an -# octal number, and 08 and 09 are illegal ocatal numbers for which the -# arithmetic operations below will fail. hh=$((10#$hh)) if [ $(($hh%2)) = 1 ]; then hh=$((hh-1)) fi -# Now that the arithmetic is done, recast hh as a two-digit string because -# that is needed in constructing the names below. + # Archive files use 2-digit forecast hour hh=$( printf "%02d\n" $hh ) arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" @@ -921,17 +625,16 @@ bination of external model (extrn_mdl_name) and analysis or forecast arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" arcv_fps="${arcv_dir}/${arcv_fns}" arcvrel_dir="" -# -# Reset hh to its original value in case it is used again later below. -# + + # Reset hh to its original value hh=${hh_orig} ;; "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# + # + # Note that this is GSL HRRRX data, not operational NCEP HRRR data. + # An option for the latter may be added in the future. + # arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" arcv_fmt="zip" arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" @@ -954,23 +657,23 @@ Archive file information has not been specified for this external model: ;; esac -# -# Depending on the experiment configuration, the above code may set -# arcv_fns and arcv_fps to either scalars or arrays. If they are not -# arrays, recast them as arrays because that is what is expected in the -# code below. -# + # + # Depending on the experiment configuration, the above code may set + # arcv_fns and arcv_fps to either scalars or arrays. If they are not + # arrays, recast them as arrays because that is what is expected in + # the code below. + # is_array arcv_fns || arcv_fns=( "${arcv_fns}" ) is_array arcv_fps || arcv_fps=( "${arcv_fps}" ) -# -#----------------------------------------------------------------------- -# -# Use the eval function to set the output variables. Note that each of -# these is set only if the corresponding input variable specifying the -# name to use for the output variable is not empty. -# -#----------------------------------------------------------------------- -# + # + #----------------------------------------------------------------------- + # + # Use the eval function to set the output variables. Note that each + # of these is set only if the corresponding input variable specifying + # the name to use for the output variable is not empty. + # + #----------------------------------------------------------------------- + # if [ ! -z "${varname_extrn_mdl_cdate}" ]; then eval ${varname_extrn_mdl_cdate}="${cdate}" fi @@ -1011,12 +714,12 @@ Archive file information has not been specified for this external model: if [ ! -z "${varname_extrn_mdl_arcvrel_dir}" ]; then eval ${varname_extrn_mdl_arcvrel_dir}="${arcvrel_dir}" fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# + # + #----------------------------------------------------------------------- + # + # Restore the shell options saved at the beginning of this script/function. + # + #----------------------------------------------------------------------- + # { restore_shell_opts; } > /dev/null 2>&1 } diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index c40e3a1280..e315c240ba 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -1,81 +1,52 @@ # #----------------------------------------------------------------------- # -# This file defines and then calls a function that sets parameters -# associated with the external model used for initial conditions (ICs) -# and the one used for lateral boundary conditions (LBCs). +# This file defines and then calls a function that sets known locations +# of files on supported platforms. # #----------------------------------------------------------------------- # -function set_extrn_mdl_params() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_NAME_ICS -# that are necessary for generating initial condition (IC) and surface -# files for the FV3SAR are stored (usually for a limited time, e.g. for -# the GFS external model, 2 weeks on WCOSS and 2 days on hera). If for -# a given cycle these files are available in this system directory, they -# will be copied over to a subdirectory under the cycle directory. If -# these files are not available in the system directory, then we search -# for them elsewhere, e.g. in the mass store (HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMINgfs}" - -else - - case "${EXTRN_MDL_NAME_ICS}" in +function set_known_sys_dir() { + + # Usage: + # set_known_sys_dir model + # + # model is the name of the external model + # + local known_sys_dir model_name + + model=$1 + # + #----------------------------------------------------------------------- + # + # Set the system directory (i.e. location on disk, not on HPSS) in + # which the files generated by the external model specified by + # EXTRN_MDL_NAME_ICS that are necessary for generating initial + # condition (IC) and surface files for the FV3SAR are stored (usually + # for a limited time, e.g. for the GFS external model, 2 weeks on + # WCOSS and 2 days on hera). If for a given cycle these files are + # available in this system directory, they will be copied over to a + # subdirectory under the cycle directory. If these files are not + # available in the system directory, then we search for them + # elsewhere, e.g. in the mass store (HPSS). + # + #----------------------------------------------------------------------- + # + + # Set some default known locations on supported platforms. Not all + # platforms have known input locations + case "${model}" in "GSMGFS") case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; "ODIN") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/ywang/EPIC/GDAS/2019053000_mem001}" + known_sys_dir=/scratch/ywang/EPIC/GDAS/2019053000_mem001 ;; "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" + known_sys_dir=/glade/p/ral/jntp/UFS_CAM/COMGFS ;; "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" + known_sys_dir=/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001 ;; esac ;; @@ -83,28 +54,24 @@ else "FV3GFS") case "$MACHINE" in "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; + ;& # Fall through "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/gfs/prod}" + known_sys_dir=/gpfs/dell1/nco/ops/com/gfs/prod ;; "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch1/NCEPDEV/rstprod/com/gfs/prod}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" + known_sys_dir=/scratch1/NCEPDEV/rstprod/com/gfs/prod ;; "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/public/data/grids/gfs/nemsio}" + known_sys_dir=/public/data/grids/gfs/nemsio ;; "ODIN") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/ywang/test_runs/FV3_regional/gfs}" + known_sys_dir=/scratch/ywang/test_runs/FV3_regional/gfs ;; "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" + known_sys_dir=/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001 ;; "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" + known_sys_dir=/glade/p/ral/jntp/UFS_CAM/COMGFS} ;; esac ;; @@ -112,22 +79,9 @@ else "RAP") case "$MACHINE" in "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; + ;& # Fall through "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch2/BMC/public/data/gsd/rap/full/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/misc/whome/rtrr/rap}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" + known_sys_dir=/gpfs/hps/nco/ops/com/rap/prod ;; esac ;; @@ -135,22 +89,9 @@ else "HRRR") case "$MACHINE" in "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; + ;& # Fall through "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/misc/whome/rtrr/hrrr}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" + known_sys_dir=/gpfs/hps/nco/ops/com/hrrr/prod ;; esac ;; @@ -158,216 +99,53 @@ else "NAM") case "$MACHINE" in "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/nam/prod}" - ;; + ;& # Fall through "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/nam/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" + known_sys_dir=/gpfs/dell1/nco/ops/com/nam/prod ;; esac ;; esac -fi -# -# If EXTRN_MDL_SYSBASEDIR_ICS has not been set (not even to a null string), -# print out an error message and exit. -# -if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS+x}" ]; then - print_err_msg_exit "\ -The variable EXTRN_MDL_SYSBASEDIR_ICS specifying the system directory -in which to look for the files generated by the external model for ICs -has not been set for the current combination of machine (MACHINE) and -external model (EXTRN_MDL_NAME_ICS): - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift -# the starting time of the external model that provides lateral boundary -# conditions. -# -#----------------------------------------------------------------------- -# -case "${EXTRN_MDL_NAME_LBCS}" in - "GSMGFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "FV3GFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "RAP") - EXTRN_MDL_LBCS_OFFSET_HRS="3" - ;; - "HRRR") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "NAM") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; -esac -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_NAME_LBCS -# that are necessary for generating lateral boundary condition (LBC) files -# for the FV3SAR are stored (usually for a limited time, e.g. for the GFS -# external model, 2 weeks on WCOSS and 2 days on hera). If for a given -# cycle these files are available in this system directory, they will be -# copied over to a subdirectory under the cycle directory. If these files -# are not available in the system directory, then we search for them -# elsewhere, e.g. in the mass store (HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMINgfs}" - -else + echo $known_sys_dir +} +function set_extrn_mdl_params() { + # + #----------------------------------------------------------------------- + # + # Use known locations or COMINgfs as default, depending on RUN_ENVIR + # + #----------------------------------------------------------------------- + # + if [ "${RUN_ENVIR}" = "nco" ]; then + EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMINgfs}" + EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMINgfs}" + else + EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$(set_known_sys_dir \ + ${EXTRN_MDL_NAME_ICS})}" + EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$(set_known_sys_dir \ + ${EXTRN_MDL_NAME_LBCS})}" + fi + + # + #----------------------------------------------------------------------- + # + # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift + # the starting time of the external model that provides lateral boundary + # conditions. + # + #----------------------------------------------------------------------- + # case "${EXTRN_MDL_NAME_LBCS}" in - - "GSMGFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/ywang/EPIC/GDAS/2019053000_mem001}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - esac - ;; - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch1/NCEPDEV/rstprod/com/gfs/prod}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/public/data/grids/gfs/nemsio}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/ywang/test_runs/FV3_regional/gfs}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - esac - ;; - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch2/BMC/public/data/gsd/rap/full/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/misc/whome/rtrr/rap}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" + "RAP") + EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"3"} ;; - esac - ;; - - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat}" + "*") + EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"0"} ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/misc/whome/rtrr/hrrr}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - esac - ;; - - "NAM") - case "$MACHINE" in - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - esac - ;; - esac - -fi -# -# If EXTRN_MDL_SYSBASEDIR_LBCS has not been set (not even to a null string), -# print out an error message and exit. -# -if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS+x}" ]; then - print_err_msg_exit "\ -The variable EXTRN_MDL_SYSBASEDIR_LBCS specifying the system directory -in which to look for the files generated by the external model for LBCs -has not been set for the current combination of machine (MACHINE) and -external model (EXTRN_MDL_NAME_LBCS): - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" -fi } # #----------------------------------------------------------------------- From b38acc43c620bc3c63feaf8ceddb7cb827ae1cbe Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:24:27 -0700 Subject: [PATCH 113/203] Enhance ability to use template variables (#650) ## DESCRIPTION OF CHANGES: 1. Enhance ability to use template variables in the experiment configuration file (either in the default configuration file `config_defaults.sh` or the user configuration file `config.sh`). 2. Modify WE2E test system to include test of template variable use. 3. Fix bugs. ### Notes on template variables: A template variable (or simply a template) is an experiment variable that contains in its definition a reference to another variable(s). The referenced variable can be another experiment variable (i.e. one that is defined in `var_defns.sh`), or it can be a local variable (i.e. one that is not defined in `var_defns.sh` but in the script or function that sources `var_defns.sh` and uses the template). For example, a template named `TEMPL_VAR` my be defined in `config_defaults.sh` or `config.sh` as `TEMPL_VAR='cd ${some_dir}'` where `some_dir` may be an experiment variable or a local variable. `TEMPL_VAR` can then be evaluated using bash's `eval` built-in command in a script or function that first sources `var_defns.sh` and, if necessary, defines `some_dir`. Note that single quotes must be used on the right-hand side to avoid expansion of `${some_dir}` before run time (i.e. when `eval` is called on `TEMPL_VAR`). For details, see the documentation added in PR #[198](https://github.com/ufs-community/ufs-srweather-app/pull/198). ### Changes to WE2E tests: * Modify the WE2E test configuration file `config.deactivate_tasks.sh` to include template variables. `deactivate_tasks` now serves as a test of both deactivating tasks and of using template variables. * Add `template_vars` as an alternate test name for `deactivate_tasks` (by creating a symlink named `config.template_vars.sh` that points to `config.deactivate_tasks.sh`). ### Bug fixes: * In `get_WE2Etest_names_subdirs_descs.sh`, change the variable `alt_test_subdirs` to `alt_test_names` at a single location. * In `setup.sh`, set `BUILD_ENV_FN` and `WFLOW_ENV_FN` (instead of in `load_modules_run_task.sh` and `launch_FV3LAM_wflow.sh`, respectively). This way, these variables will have the correct values in `var_defns.sh`. * In `get_expts_status.sh`, fix the way `homerrfs` is calculated. ## TESTS CONDUCTED: The WE2E tests `grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2` and `template_vars` were run on Hera. Both completed successfully. ## DOCUMENTATION: Documentation is added to the User's Guide via PR #[198](https://github.com/ufs-community/ufs-srweather-app/pull/198) into the ufs-srweather-app repo. ## Dependencies: PR #[198](https://github.com/ufs-community/ufs-srweather-app/pull/198) for the documentation. ## CONTRIBUTORS: @christinaholtNOAA and @mkavulich brought up the issue of templates as part of PR #[617](https://github.com/NOAA-EMC/regional_workflow/pull/617). --- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 4 +- tests/WE2E/get_expts_status.sh | 2 +- tests/WE2E/run_WE2E_tests.sh | 4 +- .../wflow_features/config.deactivate_tasks.sh | 46 +- .../wflow_features/config.template_vars.sh | 1 + ush/bash_utils/check_var_valid_value.sh | 28 +- ush/bash_utils/get_bash_file_contents.sh | 71 ++ ush/bash_utils/print_input_args.sh | 2 +- ush/check_expt_config_vars.sh | 110 +++ ush/compare_config_scripts.sh | 151 ---- ush/config_defaults.sh | 29 +- ush/generate_FV3LAM_wflow.sh | 4 +- ush/launch_FV3LAM_wflow.sh | 127 +-- ush/load_modules_run_task.sh | 3 +- ush/setup.sh | 742 ++++++++---------- ush/source_util_funcs.sh | 9 + 16 files changed, 685 insertions(+), 648 deletions(-) create mode 120000 tests/WE2E/test_configs/wflow_features/config.template_vars.sh create mode 100644 ush/bash_utils/get_bash_file_contents.sh create mode 100644 ush/check_expt_config_vars.sh delete mode 100644 ush/compare_config_scripts.sh diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 257cd89129..2183ead891 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -759,7 +759,7 @@ This is probably because it is a directory. Please correct and rerun." test_names=("${prim_test_names[@]}") test_subdirs=("${prim_test_subdirs[@]}") if [ "${num_alt_tests}" -gt "0" ]; then - test_names+=("${alt_test_subdirs[@]:-}") + test_names+=("${alt_test_names[@]:-}") test_subdirs+=("${alt_test_subdirs[@]:-}") fi # @@ -1025,7 +1025,7 @@ Please correct and rerun." # listed first. # # Finally, we extract from test_ids_and_inds_sorted the second number -# in each element (the one afte the first number, which is the test ID, +# in each element (the one after the first number, which is the test ID, # and the test type, which we no longer need), which is the original # array index before sorting, and save the results in the array sort_inds. # This array will contain the original indices in sorted order that we diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 5c05acafb1..997bfb6b8f 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -50,7 +50,7 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -homerrfs=${scrfunc_dir%/*} +homerrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index b8eaeec74f..dd1f506133 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -538,8 +538,8 @@ accordingly and rerun." if [ "${match_found}" = "FALSE" ]; then avail_WE2E_test_names_str=$( printf " \"%s\"\n" "${avail_WE2E_test_names[@]}" ) print_err_msg_exit "\ -The name current user-specified test to run (user_spec_test) does not -match any of the names (either primary or alternate) of the available +The name of the current user-specified test to run (user_spec_test) does +not match any of the names (either primary or alternate) of the available WE2E tests: user_spec_test = \"${user_spec_test}\" Valid values for user_spec_test consist of the names (primary or alternate) diff --git a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh index 52aad5cfa6..2375a648f5 100644 --- a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh +++ b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh @@ -2,10 +2,26 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test ensures that the various workflow tasks can be deactivated, -# i.e. removed from the Rocoto XML. Note that we leave the MAKE_GRID_TN, -# MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because there is a -# separate test for turning those off. +# This test has two purposes: +# +# 1) It checks that the various workflow tasks can be deactivated, i.e. +# removed from the Rocoto XML. +# 2) It checks the capability of the workflow to use "template" experiment +# variables, i.e. variables whose definitions include references to +# other variables, e.g. +# +# MY_VAR='\${ANOTHER_VAR}' +# +# Note that we do not deactivate all tasks in the workflow; we leave the +# MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because: +# +# 1) There is already a WE2E test that runs with these three tasks +# deactivated (that test is to ensure that pre-generated grid, +# orography, and surface climatology files can be used). +# 2) In checking the template variable capability, we want to make sure +# that the variable defintions file (GLOBAL_VAR_DEFNS_FN) generated +# does not have syntax or other errors in it by sourcing it in these +# three tasks. # RUN_ENVIR="community" @@ -14,13 +30,31 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" -DATE_FIRST_CYCL="20190615" -DATE_LAST_CYCL="20190615" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + RUN_TASK_GET_EXTRN_ICS="FALSE" RUN_TASK_GET_EXTRN_LBCS="FALSE" RUN_TASK_MAKE_ICS="FALSE" RUN_TASK_MAKE_LBCS="FALSE" RUN_TASK_RUN_FCST="FALSE" RUN_TASK_RUN_POST="FALSE" +# +# The following shows examples of how to define template variables. Here, +# we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template +# variables. Note that during this test, these templates aren't actually +# expanded/used (something that would be done using bash's "eval" built-in +# command) anywhere in the scripts. They are included here only to verify +# that the test completes with some variables defined as templates. +# +RUN_CMD_UTILS='cd $yyyymmdd' +RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' +RUN_CMD_POST='echo hello $yyyymmdd' diff --git a/tests/WE2E/test_configs/wflow_features/config.template_vars.sh b/tests/WE2E/test_configs/wflow_features/config.template_vars.sh new file mode 120000 index 0000000000..80ede54374 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.template_vars.sh @@ -0,0 +1 @@ +config.deactivate_tasks.sh \ No newline at end of file diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 7a0e20707b..576ad6b1b8 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -108,24 +108,34 @@ The value specified in ${var_name} is not supported: # #----------------------------------------------------------------------- # -# Check whether var_value is equal to one of the elements of the array -# valid_var_values. If not, print out an error message and exit the -# calling script. +# If var_value contains a dollar sign, we assume the corresponding variable +# (var_name) is a template variable, i.e. one whose value contains a +# reference to another variable, e.g. +# +# MY_VAR='\${ANOTHER_VAR}' +# +# In this case, we do nothing since it does not make sense to check +# whether var_value is a valid value (since its contents have not yet +# been expanded). If var_value doesn't contain a dollar sign, it must +# contain a literal string. In this case, we check whether it is equal +# to one of the elements of the array valid_var_values. If not, we +# print out an error message and exit the calling script. # #----------------------------------------------------------------------- # - is_element_of "valid_var_values" "${var_value}" || { \ - valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); - print_err_msg_exit "\ + if [[ "${var_value}" != *'$'* ]]; then + is_element_of "valid_var_values" "${var_value}" || { \ + valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); + print_err_msg_exit "\ ${err_msg} ${var_name} must be set to one of the following: ${valid_var_values_str}"; \ - } + } + fi # #----------------------------------------------------------------------- # -# Restore the shell options saved at the beginning of this script/func- -# tion. +# Restore the shell options saved at the beginning of this script/function. # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/get_bash_file_contents.sh b/ush/bash_utils/get_bash_file_contents.sh new file mode 100644 index 0000000000..3b3ab7b30d --- /dev/null +++ b/ush/bash_utils/get_bash_file_contents.sh @@ -0,0 +1,71 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that returns the contents of a bash script/ +# function with all empty lines, comment lines, and leading and trailing +# whitespace removed. Arguments are as follows: +# +# fp: +# The relative or full path to the file containing the bash script or +# function. +# +# output_varname_contents: +# Name of the output variable that will contain the (processed) contents +# of the file. This is the output of the function. +# +#----------------------------------------------------------------------- +# +function get_bash_file_contents() { + + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + + local valid_args=( \ + "fp" \ + "output_varname_contents" \ + ) + process_args valid_args "$@" + print_input_args "valid_args" + # + # Verify that the required arguments to this function have been specified. + # If not, print out an error message and exit. + # + if [ -z "$fp" ]; then + print_err_msg_exit "\ +The argument \"fp\" specifying the relative or full path to the file to +read was not specified in the call to this function: + fp = \"$fp\"" + fi + + local contents \ + crnt_line + # + # Read in all lines in the file. In doing so: + # + # 1) Concatenate any line ending with the bash line continuation character + # (a backslash) with the following line. + # 2) Remove any leading and trailing whitespace. + # + # Note that these two actions are automatically performed by the "read" + # utility in the while-loop below. + # + contents="" + while read crnt_line; do + contents="${contents}${crnt_line} +" + done < "$fp" + # + # Strip out any comment and empty lines from contents. + # + contents=$( printf "${contents}" | \ + $SED -r -e "/^#.*/d" `# Remove comment lines.` \ + -e "/^$/d" `# Remove empty lines.` \ + ) + # + # Set output variables. + # + printf -v ${output_varname_contents} "${contents}" + + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 957cec5243..d5ba5bd53c 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -169,7 +169,7 @@ have been set as follows: #----------------------------------------------------------------------- # # If a global variable named DEBUG is not defined, print out the message. -# If it is defined, print out the message only if DEBUG is set to TRUE. +# If it is defined, print out the message only if DEBUG is set to "TRUE". # #----------------------------------------------------------------------- # diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh new file mode 100644 index 0000000000..53ce13a094 --- /dev/null +++ b/ush/check_expt_config_vars.sh @@ -0,0 +1,110 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that checks that all experiment variables +# set in the user-specified experiment configuration file are defined (by +# being assigned default values) in the default experiment configuration +# file. If a variable is found in the former that is not defined in the +# latter, this function exits with an error message. +# +# This check is performed in order to prevent the user from defining +# arbitrary variables in the user-specified configuration file; the +# latter should be used to specify only varaibles that have already been +# defined in the default configuration file. +# +# Arguments are as follows: +# +# default_config_fp: +# The relative or full path to the default experiment configuration file. +# +# config_fp: +# The relative or full path to the user-specified experiment configuration +# file. +# +#----------------------------------------------------------------------- +# +function check_expt_config_vars() { + + . ${scrfunc_dir}/source_util_funcs.sh + + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + + local valid_args=( \ + "default_config_fp" \ + "config_fp" \ + ) + process_args valid_args "$@" + print_input_args "valid_args" + + local var_list_default \ + var_list_user \ + crnt_line \ + var_name \ + regex_search + # + # Get the list of variable definitions, first from the default experiment + # configuration file and then from the user-specified experiment + # configuration file. + # + get_bash_file_contents fp="${default_config_fp}" \ + output_varname_contents="var_list_default" + + get_bash_file_contents fp="${config_fp}" \ + output_varname_contents="var_list_user" + # + # Loop through each line/variable in var_list_user. For each line, + # extract the the name of the variable that is being set (say VAR) and + # check that this variable is set somewhere in the default configuration + # file by verifying that a line that starts with "VAR=" exists in + # var_list_default. + # + while read crnt_line; do + # + # Note that a variable name will be found only if the equal sign immediately + # follows the variable name. + # + var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") + + if [ -z "${var_name}" ]; then + + print_info_msg " +The current line (crnt_line) of the user-specified experiment configuration +file (config_fp) does not contain a variable name (i.e. var_name is empty): + config_fp = \"${config_fp}\" + crnt_line = \"${crnt_line}\" + var_name = \"${var_name}\" +Skipping to next line." + + else + # + # Use grep to search for the variable name (followed by an equal sign, + # all at the beginning of a line) in the list of variables in the default + # configuration file. + # + # Note that we use a herestring to input into grep the list of variables + # in the default configuration file. grep will return with a zero status + # if the specified string (regex_search) is not found in the default + # variables list and a nonzero status otherwise. Note also that we + # redirect the output of grep to null because we are only interested in + # its exit status. + # + regex_search="^${var_name}=" + grep "${regex_search}" <<< "${var_list_default}" > /dev/null 2>&1 || \ + print_err_msg_exit "\ +The variable (var_name) defined on the current line (crnt_line) of the +user-specified experiment configuration file (config_fp) does not appear +in the default experiment configuration file (default_config_fp): + config_fp = \"${config_fp}\" + default_config_fp = \"${default_config_fp}\" + crnt_line = \"${crnt_line}\" + var_name = \"${var_name}\" +Please assign a default value to this variable in the default configuration +file and rerun." + + fi + + done <<< "${var_list_user}" + + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/compare_config_scripts.sh b/ush/compare_config_scripts.sh deleted file mode 100644 index 791fa4e340..0000000000 --- a/ush/compare_config_scripts.sh +++ /dev/null @@ -1,151 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that checks that all vari- -# ables defined in the user-specified experiment/workflow configuration -# file (whose file name is stored in the variable EXPT_CONFIG_FN) are -# also assigned default values in the default configuration file (whose -# file name is stored in the variable EXPT_DEFAULT_CONFIG_FN). -# -#----------------------------------------------------------------------- -# -function compare_config_scripts() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Source bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${scrfunc_dir}/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Create a list of variable settings in the default workflow/experiment -# file (script) by stripping out comments, blank lines, extraneous lead- -# ing whitespace, etc from that file and saving the result in the varia- -# ble var_list_default. Each line of var_list_default will have the -# form -# -# VAR=... -# -# where the VAR is a variable name and ... is the value (including any -# trailing comments). Then create an equivalent list for the local con- -# figuration file and save the result in var_list_local. -# -#----------------------------------------------------------------------- -# -var_list_default=$( \ -$SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${EXPT_DEFAULT_CONFIG_FN} \ -) - -var_list_local=$( \ -$SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${EXPT_CONFIG_FN} \ -) -# -#----------------------------------------------------------------------- -# -# Loop through each line of var_list_local. For each line, extract the -# the name of the variable that is being set (say VAR) and check that -# this variable is set somewhere in the default configuration file by -# verifying that a line that starts with "VAR=" exists in var_list_de- -# fault. -# -#----------------------------------------------------------------------- -# -while read crnt_line; do -# -# Note that a variable name will be found only if the equal sign immed- -# iately follows the variable name. -# - var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") - - if [ -z "${var_name}" ]; then - - print_info_msg " -Current line (crnt_line) of user-specified experiment/workflow configu- -ration file (EXPT_CONFIG_FN) does not contain a variable name (i.e. -var_name is empty): - EXPT_CONFIG_FN = \"${EXPT_CONFIG_FN}\" - crnt_line = \"${crnt_line}\" - var_name = \"${var_name}\" -Skipping to next line." - - else -# -# Use a herestring to input list of variables in the default configura- -# tion file to grep. Also, redirect the output to null because we are -# only interested in the exit status of grep (which will be nonzero if -# the specified regex was not found in the list).. -# - grep "^${var_name}=" <<< "${var_list_default}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -The variable specified by var_name in the user-specified experiment/ -workflow configuration file (EXPT_CONFIG_FN) does not appear in the de- -fault experiment/workflow configuration file (EXPT_DEFAULT_CONFIG_FN): - EXPT_CONFIG_FN = \"${EXPT_CONFIG_FN}\" - EXPT_DEFAULT_CONFIG_FN = \"${EXPT_DEFAULT_CONFIG_FN}\" - var_name = \"${var_name}\" -Please assign a default value to this variable in the default configura- -tion file and rerun." - - fi - -done <<< "${var_list_local}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - -} -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# -compare_config_scripts - diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 21d6096a16..afb9428567 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -152,13 +152,14 @@ QUEUE_FCST="" # will be ignored unless WORKFLOW_MANAGER="none". Definitions: # # RUN_CMD_UTILS: -# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, etc.) -# Can be left blank for smaller domains, in which case the executables will run -# without MPI. +# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, +# etc.) Can be left blank for smaller domains, in which case the executables +# will run without MPI. # # RUN_CMD_FCST: -# The run command for the model forecast step. This will be appended to the end -# of the variable definitions file, so it can reference other variables. +# The run command for the model forecast step. This will be appended to +# the end of the variable definitions file, so it can reference other +# variables. # # RUN_CMD_POST: # The run command for post-processing (UPP). Can be left blank for smaller @@ -167,7 +168,7 @@ QUEUE_FCST="" #----------------------------------------------------------------------- # RUN_CMD_UTILS="mpirun -np 1" -RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}" +RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' RUN_CMD_POST="mpirun -np 1" # #----------------------------------------------------------------------- @@ -295,7 +296,7 @@ PTMP="/base/path/of/directory/containing/postprocessed/output/files" # #----------------------------------------------------------------------- # -# Set the sparator character(s) to use in the names of the grid, mosaic, +# Set the separator character(s) to use in the names of the grid, mosaic, # and orography fixed files. # # Ideally, the same separator should be used in the names of these fixed @@ -732,6 +733,7 @@ NOMADS_file_type="nemsio" # directory or the cycle directories under it. # #----------------------------------------------------------------------- +# CCPP_PHYS_SUITE="FV3_GFS_v15p2" # #----------------------------------------------------------------------- @@ -1754,9 +1756,9 @@ NUM_ENS_MEMBERS="1" # #----------------------------------------------------------------------- # -DO_SHUM="false" -DO_SPPT="false" -DO_SKEB="false" +DO_SHUM="FALSE" +DO_SPPT="FALSE" +DO_SKEB="FALSE" SHUM_MAG="0.006" #Variable "shum" in input.nml SHUM_LSCALE="150000" SHUM_TSCALE="21600" #Variable "shum_tau" in input.nml @@ -1770,7 +1772,7 @@ SKEB_LSCALE="150000" SKEB_TSCALE="21600" #Variable "skeb_tau" in input.nml SKEB_INT="3600" #Variable "skebint" in input.nml SKEB_VDOF="10" -USE_ZMTNBLCK="false" +USE_ZMTNBLCK="FALSE" # #----------------------------------------------------------------------- # @@ -1787,7 +1789,7 @@ USE_ZMTNBLCK="false" # #----------------------------------------------------------------------- # -DO_SPP="false" +DO_SPP="FALSE" SPP_VAR_LIST=( "pbl" ) SPP_MAG_LIST=( "0.2" ) #Variable "spp_prt_list" in input.nml SPP_LSCALE=( "150000.0" ) @@ -1902,6 +1904,3 @@ OMP_STACKSIZE_RUN_FCST="1024m" KMP_AFFINITY_RUN_POST="scatter" OMP_NUM_THREADS_RUN_POST="1" OMP_STACKSIZE_RUN_POST="1024m" -# -#----------------------------------------------------------------------- -# diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index db65baec21..4776c3d455 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -1107,8 +1107,8 @@ if [[ $retval == 0 ]]; then # else printf " -Experiment/workflow generation failed. Check the log file from the ex- -periment/workflow generation script in the file specified by log_fp: +Experiment generation failed. Check the log file from the experiment +generation script in the file specified by log_fp: log_fp = \"${log_fp}\" Stopping. " diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 76c964083a..4c881be2db 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -18,7 +18,13 @@ set -u #----------------------------------------------------------------------- # if [[ $(uname -s) == Darwin ]]; then - command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + command -v greadlink >/dev/null 2>&1 || { \ + echo >&2 "\ +For Darwin-based operating systems (MacOS), the 'greadlink' utility is +required to run the UFS SRW Application. Reference the User's Guide for +more information about platform requirements. Aborting."; \ + exit 1; \ + } scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) else scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) @@ -28,27 +34,26 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) ushdir="${scrfunc_dir}" . $ushdir/source_util_funcs.sh - # #----------------------------------------------------------------------- # # Get the experiment directory. We assume that there is a symlink to # this script in the experiment directory, and this script is called via -# that symlink. Thus, finding the directory in which the symlink is lo- -# cated will give us the experiment directory. We find this by first +# that symlink. Thus, finding the directory in which the symlink is +# located will give us the experiment directory. We find this by first # obtaining the directory portion (i.e. the portion without the name of # this script) of the command that was used to called this script (i.e. -# "$0") and then use the "readlink -f" command to obtain the correspond- -# ing absolute path. This will work for all four of the following ways -# in which the symlink in the experiment directory pointing to this -# script may be called: +# "$0") and then use the "readlink -f" command to obtain the corresponding +# absolute path. This will work for all four of the following ways in +# which the symlink in the experiment directory pointing to this script +# may be called: # # 1) Call this script from the experiment directory: # > cd /path/to/experiment/directory # > launch_FV3LAM_wflow.sh # -# 2) Call this script from the experiment directory but using "./" be- -# fore the script name: +# 2) Call this script from the experiment directory but using "./" before +# the script name: # > cd /path/to/experiment/directory # > ./launch_FV3LAM_wflow.sh # @@ -56,9 +61,8 @@ ushdir="${scrfunc_dir}" # symlink in the experiment directory: # > /path/to/experiment/directory/launch_FV3LAM_wflow.sh # -# 4) Call this script from a directory that is several levels up from -# the experiment directory (but not necessarily at the root directo- -# ry): +# 4) Call this script from a directory that is several levels up from the +# experiment directory (but not necessarily at the root directory): # > cd /path/to # > experiment/directory/launch_FV3LAM_wflow.sh # @@ -75,7 +79,13 @@ ushdir="${scrfunc_dir}" # exptdir=$( dirname "$0" ) if [[ $(uname -s) == Darwin ]]; then - command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + command -v greadlink >/dev/null 2>&1 || { \ + echo >&2 "\ +For Darwin-based operating systems (MacOS), the 'greadlink' utility is +required to run the UFS SRW Application. Reference the User's Guide for +more information about platform requirements. Aborting."; + exit 1; + } exptdir=$( greadlink -f "$exptdir" ) else exptdir=$( readlink -f "$exptdir" ) @@ -106,14 +116,12 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -machine=$(echo_lowercase $MACHINE) -env_fn=${WFLOW_ENV_FN:-"wflow_${machine}.env"} -env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" +env_fp="${SR_WX_APP_TOP_DIR}/env/${WFLOW_ENV_FN}" module purge source "${env_fp}" || print_err_msg_exit "\ - Sourcing platform-specific environment file (env_fp) for -the workflow task failed : -env_fp = \"${env_fp}\"" +Sourcing platform-specific environment file (env_fp) for the workflow +task failed: + env_fp = \"${env_fp}\"" # #----------------------------------------------------------------------- # @@ -141,20 +149,19 @@ wflow_status="IN PROGRESS" # #----------------------------------------------------------------------- # -cd "$exptdir" +cd_vrfy "$exptdir" # #----------------------------------------------------------------------- # -# Issue the rocotorun command to (re)launch the next task in the -# workflow. Then check for error messages in the output of rocotorun. -# If any are found, it means the end-to-end run of the workflow failed. -# In this case, we remove the crontab entry that launches the workflow, -# and we append an appropriate failure message at the end of the launch -# log file. +# Issue the rocotorun command to (re)launch the next task in the workflow. +# Then check for error messages in the output of rocotorun. If any are +# found, it means the end-to-end run of the workflow failed. In this +# case, we remove the crontab entry that launches the workflow, and we +# append an appropriate failure message at the end of the launch log +# file. # #----------------------------------------------------------------------- # - tmp_fn="rocotorun_output.txt" rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 @@ -239,15 +246,15 @@ ${rocotostat_output} # ... # # Thus, the first row is a header line containing the column titles, and -# the remaining rows each correspond to one cycle in the workflow. Be- -# low, we are interested in the first and second columns of each row. -# The first column is a string containing the start time of the cycle -# (in the format YYYYMMDDHHmm, where YYYY is the 4-digit year, MM is the -# 2-digit month, DD is the 2-digit day of the month, HH is the 2-digit -# hour of the day, and mm is the 2-digit minute of the hour). The se- -# cond column is a string containing the state of the cycle. This can -# be "Active" or "Done". Below, we read in and store these two columns -# in (1-D) arrays. +# the remaining rows each correspond to one cycle in the workflow. Below, +# we are interested in the first and second columns of each row. The +# first column is a string containing the start time of the cycle (in the +# format YYYYMMDDHHmm, where YYYY is the 4-digit year, MM is the 2-digit +# month, DD is the 2-digit day of the month, HH is the 2-digit hour of +# the day, and mm is the 2-digit minute of the hour). The second column +# is a string containing the state of the cycle. This can be "Active" +# or "Done". Below, we read in and store these two columns in (1-D) +# arrays. # #----------------------------------------------------------------------- # @@ -259,9 +266,9 @@ cycle_status=() i=0 while read -r line; do # -# Note that the first line in rocotostat_output is a header line con- -# taining the column titles. Thus, we ignore it and consider only the -# remaining lines (of which there is one per cycle). +# Note that the first line in rocotostat_output is a header line containing +# the column titles. Thus, we ignore it and consider only the remaining +# lines (of which there is one per cycle). # if [ $i -gt 0 ]; then im1=$((i-1)) @@ -289,9 +296,9 @@ done # #----------------------------------------------------------------------- # -# If the number of completed cycles is equal to the total number of cy- -# cles, it means the end-to-end run of the workflow was successful. In -# this case, we reset the wflow_status to "SUCCESS". +# If the number of completed cycles is equal to the total number of cycles, +# it means the end-to-end run of the workflow was successful. In this +# case, we reset the wflow_status to "SUCCESS". # #----------------------------------------------------------------------- # @@ -301,8 +308,8 @@ fi # #----------------------------------------------------------------------- # -# Print informational messages about the workflow to the launch log -# file, including the workflow status. +# Print informational messages about the workflow to the launch log file, +# including the workflow status. # #----------------------------------------------------------------------- # @@ -333,25 +340,24 @@ if [ "${wflow_status}" = "SUCCESS" ] || \ msg=" The end-to-end run of the workflow for the forecast experiment specified -by expt_name has completed with the following workflow status (wflow_- -status): +by expt_name has completed with the following workflow status (wflow_status): expt_name = \"${expt_name}\" wflow_status = \"${wflow_status}\" " # # If a cron job was being used to periodically relaunch the workflow, we -# now remove the entry in the crontab corresponding to the workflow be- -# cause the end-to-end run of the workflow has now either succeeded or +# now remove the entry in the crontab corresponding to the workflow +# because the end-to-end run of the workflow has now either succeeded or # failed and will remain in that state without manual user intervention. -# Thus, there is no need to try to relaunch it. We also append a mes- -# sage to the completion message above to indicate this. +# Thus, there is no need to try to relaunch it. We also append a message +# to the completion message above to indicate this. # if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then msg="${msg}\ -Thus, there is no need to relaunch the workflow via a cron job. Remo- -ving from the crontab the line (CRONTAB_LINE) that calls the workflow -launch script for this experiment: +Thus, there is no need to relaunch the workflow via a cron job. Removing +from the crontab the line (CRONTAB_LINE) that calls the workflow launch +script for this experiment: CRONTAB_LINE = \"${CRONTAB_LINE}\" " # @@ -364,12 +370,13 @@ launch script for this experiment: $SED -r -e "s%[*]%\\\\*%g" ) # # In the string passed to the grep command below, we use the line start -# and line end anchors ("^" and "$", respectively) to ensure that we on- -# ly find lines in the crontab that contain exactly the string in cron- -# tab_line_esc_astr without any leading or trailing characters. +# and line end anchors ("^" and "$", respectively) to ensure that we +# only find lines in the crontab that contain exactly the string in +# crontab_line_esc_astr without any leading or trailing characters. # if [ "$MACHINE" = "WCOSS_DELL_P3" ];then - grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" > tmpfile && mv tmpfile "/u/$USER/cron/mycrontab" + grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" \ + > tmpfile && mv_vrfy tmpfile "/u/$USER/cron/mycrontab" else ( crontab -l | grep -v "^${crontab_line_esc_astr}$" ) | crontab - fi @@ -388,7 +395,3 @@ launch script for this experiment: fi fi - - - - diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 1f0d881260..17b6d5d3d4 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -135,8 +135,7 @@ jjob_fp="$2" module purge machine=$(echo_lowercase $MACHINE) -env_fn=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} -env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" +env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" source "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: diff --git a/ush/setup.sh b/ush/setup.sh index 88dcab21e5..d8db064c69 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -63,6 +63,7 @@ Starting function ${func_name}() in \"${scrfunc_fn}\"... # #----------------------------------------------------------------------- # +. ./check_expt_config_vars.sh . ./set_cycle_dates.sh . ./set_gridparams_GFDLgrid.sh . ./set_gridparams_ESGgrid.sh @@ -111,7 +112,9 @@ if [ -f "${EXPT_CONFIG_FN}" ]; then # configuration file are also assigned default values in the default # configuration file. # - . ./compare_config_scripts.sh + check_expt_config_vars \ + default_config_fp="./${EXPT_DEFAULT_CONFIG_FN}" \ + config_fp="./${EXPT_CONFIG_FN}" # # Now source the user-specified configuration file. # @@ -698,12 +701,27 @@ fi # #----------------------------------------------------------------------- # -# Calculate PPN_RUN_FCST from NCORES_PER_NODE and OMP_NUM_THREADS_RUN_FCST +# Set the names of the build and workflow environment files (if not +# already specified by the user). These are the files that need to be +# sourced before building the component SRW App codes and running various +# workflow scripts, respectively. # #----------------------------------------------------------------------- # -PPN_RUN_FCST_OPT="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" -PPN_RUN_FCST=${PPN_RUN_FCST:-${PPN_RUN_FCST_OPT}} +machine=$(echo_lowercase ${MACHINE}) +WFLOW_ENV_FN=${WFLOW_ENV_FN:-"wflow_${machine}.env"} +BUILD_ENV_FN=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} +# +#----------------------------------------------------------------------- +# +# Calculate a default value for the number of processes per node for the +# RUN_FCST_TN task. Then set PPN_RUN_FCST to this default value if +# PPN_RUN_FCST is not already specified by the user. +# +#----------------------------------------------------------------------- +# +ppn_run_fcst_default="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" +PPN_RUN_FCST=${PPN_RUN_FCST:-${ppn_run_fcst_default}} # #----------------------------------------------------------------------- # @@ -882,32 +900,32 @@ fi # #----------------------------------------------------------------------- # -CYCL_HRS_str=$(printf "\"%s\" " "${CYCL_HRS[@]}") -CYCL_HRS_str="( $CYCL_HRS_str)" +cycl_hrs_str=$(printf "\"%s\" " "${CYCL_HRS[@]}") +cycl_hrs_str="( ${cycl_hrs_str})" i=0 -for CYCL in "${CYCL_HRS[@]}"; do +for cycl_hr in "${CYCL_HRS[@]}"; do - CYCL_OR_NULL=$( printf "%s" "$CYCL" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) + cycl_hr_or_null=$( printf "%s" "${cycl_hr}" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) - if [ -z "${CYCL_OR_NULL}" ]; then + if [ -z "${cycl_hr_or_null}" ]; then print_err_msg_exit "\ Each element of CYCL_HRS must be a string consisting of exactly 2 digits -(including a leading \"0\", if necessary) specifying an hour-of-day. Ele- -ment #$i of CYCL_HRS (where the index of the first element is 0) does not -have this form: - CYCL_HRS = $CYCL_HRS_str +(including a leading \"0\", if necessary) specifying an hour-of-day. +Element #$i of CYCL_HRS (where the index of the first element is 0) does +not have this form: + CYCL_HRS = ${cycl_hrs_str} CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" fi - if [ "${CYCL_OR_NULL}" -lt "0" ] || \ - [ "${CYCL_OR_NULL}" -gt "23" ]; then + if [ "${cycl_hr_or_null}" -lt "0" ] || \ + [ "${cycl_hr_or_null}" -gt "23" ]; then print_err_msg_exit "\ -Each element of CYCL_HRS must be an integer between \"00\" and \"23\", in- -clusive (including a leading \"0\", if necessary), specifying an hour-of- -day. Element #$i of CYCL_HRS (where the index of the first element is 0) -does not have this form: - CYCL_HRS = $CYCL_HRS_str +Each element of CYCL_HRS must be an integer between \"00\" and \"23\", +inclusive (including a leading \"0\", if necessary), specifying an hour- +of-day. Element #$i of CYCL_HRS (where the index of the first element +is 0) does not have this form: + CYCL_HRS = ${cycl_hrs_str} CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" fi @@ -929,7 +947,7 @@ if [ "${INCR_CYCL_FREQ}" -lt "24" ] && [ "$i" -gt "1" ]; then The number of CYCL_HRS does not match with that expected by INCR_CYCL_FREQ: INCR_CYCL_FREQ = ${INCR_CYCL_FREQ} cycle interval by the number of CYCL_HRS = ${cycl_intv} - CYCL_HRS = $CYCL_HRS_str " + CYCL_HRS = ${cycl_hrs_str}" fi im1=$(( $i-1 )) @@ -941,7 +959,7 @@ The number of CYCL_HRS does not match with that expected by INCR_CYCL_FREQ: print_err_msg_exit "\ Element #${itmp} of CYCL_HRS does not match with the increment of cycle frequency INCR_CYCL_FREQ: - CYCL_HRS = $CYCL_HRS_str + CYCL_HRS = ${cycl_hrs_str} INCR_CYCL_FREQ = ${INCR_CYCL_FREQ} CYCL_HRS[$itmp] = \"${CYCL_HRS[$itmp]}\"" fi @@ -961,7 +979,7 @@ fi set_cycle_dates \ date_start="${DATE_FIRST_CYCL}" \ date_end="${DATE_LAST_CYCL}" \ - cycle_hrs="${CYCL_HRS_str}" \ + cycle_hrs="${cycl_hrs_str}" \ incr_cycl_freq="${INCR_CYCL_FREQ}" \ output_varname_all_cdates="ALL_CDATES" @@ -1000,8 +1018,7 @@ fi # Directory containing various executable files. # # TEMPLATE_DIR: -# Directory in which templates of various FV3-LAM input files are locat- -# ed. +# Directory in which templates of various FV3-LAM input files are located. # # UFS_WTHR_MDL_DIR: # Directory in which the (NEMS-enabled) FV3-LAM application is located. @@ -1118,7 +1135,7 @@ One or more fix file directories have not been specified for this machine: TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} -You can specify the missing location(s) in config.sh" +You can specify the missing location(s) in ${EXPT_CONFIG_FN}." fi ;; @@ -1208,7 +1225,7 @@ check_var_valid_value \ # Set USE_CUSTOM_POST_CONFIG_FILE to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase $USE_CUSTOM_POST_CONFIG_FILE) +USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase ${USE_CUSTOM_POST_CONFIG_FILE}) if [ "$USE_CUSTOM_POST_CONFIG_FILE" = "TRUE" ] || \ [ "$USE_CUSTOM_POST_CONFIG_FILE" = "YES" ]; then USE_CUSTOM_POST_CONFIG_FILE="TRUE" @@ -1441,7 +1458,7 @@ must set DT_SUBHOURLY_POST_MNTS to something other than zero." # # For now, the sub-hourly capability is restricted to having values of # DT_SUBHOURLY_POST_MNTS that evenly divide into 60 minutes. This is -# because the jinja rocoto XML template (FV3LAM_wflow.xml) assumes that +# because the jinja rocoto XML template (${WFLOW_XML_FN}) assumes that # model output is generated at the top of every hour (i.e. at 00 minutes). # This restricts DT_SUBHOURLY_POST_MNTS to the following values (inluding # both cases with and without a leading 0): @@ -1574,8 +1591,8 @@ fi # #----------------------------------------------------------------------- # -# The FV3 forecast model needs the following input files in the run di- -# rectory to start a forecast: +# The FV3 forecast model needs the following input files in the run +# directory to start a forecast: # # (1) The data table file # (2) The diagnostics table file @@ -1583,27 +1600,22 @@ fi # (4) The FV3 namelist file # (5) The model configuration file # (6) The NEMS configuration file -# -# If using CCPP, it also needs: -# # (7) The CCPP physics suite definition file # # The workflow contains templates for the first six of these files. # Template files are versions of these files that contain placeholder -# (i.e. dummy) values for various parameters. The experiment/workflow -# generation scripts copy these templates to appropriate locations in -# the experiment directory (either the top of the experiment directory -# or one of the cycle subdirectories) and replace the placeholders in -# these copies by actual values specified in the experiment/workflow -# configuration file (or derived from such values). The scripts then -# use the resulting "actual" files as inputs to the forecast model. -# -# Note that the CCPP physics suite defintion file does not have a cor- -# responding template file because it does not contain any values that -# need to be replaced according to the experiment/workflow configura- -# tion. If using CCPP, this file simply needs to be copied over from -# its location in the forecast model's directory structure to the ex- -# periment directory. +# (i.e. dummy) values for various parameters. The experiment generation +# and/or the forecast task (i.e. J-job) scripts copy these templates to +# appropriate locations in the experiment directory (e.g. to the top of +# the experiment directory, to one of the cycle subdirectories, etc) and +# replace the placeholders with actual values to obtain the files that +# are used as inputs to the forecast model. +# +# Note that the CCPP physics suite defintion file (SDF) does not have a +# corresponding template file because it does not contain any values +# that need to be replaced according to the experiment configuration. +# This file simply needs to be copied over from its location in the +# forecast model's directory structure to the experiment directory. # # Below, we first set the names of the templates for the first six files # listed above. We then set the full paths to these template files. @@ -2057,7 +2069,6 @@ Reset value is:" print_info_msg "$msg" fi - # #----------------------------------------------------------------------- # @@ -2181,7 +2192,6 @@ fi #----------------------------------------------------------------------- # . ./set_extrn_mdl_params.sh - # #----------------------------------------------------------------------- # @@ -2536,7 +2546,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - # #----------------------------------------------------------------------- # @@ -2584,179 +2593,106 @@ set_thompson_mp_fix_files \ # #----------------------------------------------------------------------- # -# Generate the shell script that will appear in the experiment directory -# (EXPTDIR) and will contain definitions of variables needed by the va- -# rious scripts in the workflow. We refer to this as the experiment/ -# workflow global variable definitions file. We will create this file -# by: -# -# 1) Copying the default workflow/experiment configuration file (speci- -# fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- -# rectory specified by USHDIR) to the experiment directory and rena- -# ming it to the name specified by GLOBAL_VAR_DEFNS_FN. -# -# 2) Resetting the default variable values in this file to their current -# values. This is necessary because these variables may have been -# reset by the user-specified configuration file (if one exists in -# USHDIR) and/or by this setup script, e.g. because predef_domain is -# set to a valid non-empty value. -# -# 3) Appending to the variable definitions file any new variables intro- -# duced in this setup script that may be needed by the scripts that -# perform the various tasks in the workflow (and which source the va- -# riable defintions file). -# -# First, set the full path to the variable definitions file and copy the -# default configuration script into it. +# Set the full path to the experiment's variable definitions file. This +# file will contain definitions of variables (in bash syntax) needed by +# the various scripts in the workflow. # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP="$EXPTDIR/$GLOBAL_VAR_DEFNS_FN" -cp_vrfy $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ${GLOBAL_VAR_DEFNS_FP} +GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # #----------------------------------------------------------------------- # -# +# Get the list of primary experiment variables and their default values +# from the default experiment configuration file (EXPT_DEFAULT_CONFIG_FN). +# By "primary", we mean those variables that are defined in the default +# configuration file and can be reset in the user-specified experiment +# configuration file (EXPT_CONFIG_FN). The default values will be updated +# below to user-specified ones and the result saved in the experiment's +# variable definitions file. # #----------------------------------------------------------------------- # +print_info_msg " +Creating list of default experiment variable definitions..." -# Read all lines of GLOBAL_VAR_DEFNS file into the variable line_list. -line_list=$( $SED -r -e "s/(.*)/\1/g" ${GLOBAL_VAR_DEFNS_FP} ) -# -# Loop through the lines in line_list and concatenate lines ending with -# the line bash continuation character "\". -# -rm_vrfy ${GLOBAL_VAR_DEFNS_FP} -while read crnt_line; do - printf "%s\n" "${crnt_line}" >> ${GLOBAL_VAR_DEFNS_FP} -done <<< "${line_list}" -# -#----------------------------------------------------------------------- -# -# The following comment block needs to be updated because now line_list -# may contain lines that are not assignment statements (e.g. it may con- -# tain if-statements). Such lines are ignored in the while-loop below. -# -# Reset each of the variables in the variable definitions file to its -# value in the current environment. To accomplish this, we: -# -# 1) Create a list of variable settings by stripping out comments, blank -# lines, extraneous leading whitespace, etc from the variable defini- -# tions file (which is currently identical to the default workflow/ -# experiment configuration script) and saving the result in the vari- -# able line_list. Each line of line_list will have the form -# -# VAR=... -# -# where the VAR is a variable name and ... is the value from the de- -# fault configuration script (which does not necessarily correspond -# to the current value of the variable). -# -# 2) Loop through each line of line_list. For each line, we extract the -# variable name (and save it in the variable var_name), get its value -# from the current environment (using bash indirection, i.e. -# ${!var_name}), and use the set_file_param() function to replace the -# value of the variable in the variable definitions script (denoted -# above by ...) with its current value. -# -#----------------------------------------------------------------------- -# -# Also should remove trailing whitespace... -line_list=$( $SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${GLOBAL_VAR_DEFNS_FP} ) +get_bash_file_contents fp="$USHDIR/${EXPT_DEFAULT_CONFIG_FN}" \ + output_varname_contents="default_var_defns" print_info_msg "$DEBUG" " -Before updating default values of experiment variables to user-specified -values, the variable \"line_list\" contains: +The variable \"default_var_defns\" containing default values of primary +experiment variables is set as follows: -${line_list} +${default_var_defns} " # #----------------------------------------------------------------------- # -# Add a comment at the beginning of the variable definitions file that -# indicates that the first section of that file is (mostly) the same as -# the configuration file. -# -#----------------------------------------------------------------------- -# -read -r -d '' str_to_insert << EOM +# Create a list of primary experiment variable definitions containing +# updated values. By "updated", we mean non-default values. Values +# may have been updated due to the presence of user-specified values in +# the experiment configuration file (EXPT_CONFIG_FN) or due to other +# considerations (e.g. resetting depending on the platform the App is +# running on). # #----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 1: -# This section is a copy of the default experiment configuration file -# (${EXPT_DEFAULT_CONFIG_FN}) in the shell scripts directory specified by USHDIR -# except that variable values have been updated to those for the experiment -# (as opposed to the default values). -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# -EOM # -# Replace all occurrences of actual newlines in the variable str_to_insert -# with escaped backslash-n. This is needed for the sed command below to -# work properly (i.e. to avoid it failing with an "unterminated `s' command" -# error message). -# -str_to_insert=${str_to_insert//$'\n'/\\n} +print_info_msg " +Creating lists of (updated) experiment variable definitions..." # -# Insert str_to_insert into GLOBAL_VAR_DEFNS_FP right after the line -# containing the name of the interpreter (i.e. the line that starts with -# the string "#!", e.g. "#!/bin/bash"). +# Set the flag that specifies whether or not array variables will be +# recorded in the variable definitions file on one line or one element +# per line. Then, if writing arrays one element per line (i.e. multiline), +# set an escaped-newline character that needs to be included after every +# element of each array as the newline character in order for sed to +# write the line properly. # -regexp="(^#!.*)" -$SED -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} +multiline_arrays="TRUE" +#multiline_arrays="FALSE" +escbksl_nl_or_null="" +if [ "${multiline_arrays}" = "TRUE" ]; then + escbksl_nl_or_null='\\\n' +fi # -# Loop through the lines in line_list. +# Loop through the lines in default_var_defns. Reset the value of the +# variable on each line to the updated value (e.g. to a user-specified +# value, as opposed to the default value). The updated list of variables +# and values will be saved in var_defns. # -print_info_msg " -Generating the global experiment variable definitions file specified by -GLOBAL_VAR_DEFNS_FN: - GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" -Full path to this file is: - GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" -For more detailed information, set DEBUG to \"TRUE\" in the experiment -configuration file (\"${EXPT_CONFIG_FN}\")." - -template_var_names=() -template_var_values=() +var_defns="" while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. -# This will be successful only if the line consists of one or more char- -# acters representing the name of a variable (recall that in generating -# the variable line_list, leading spaces on each line were stripped out), -# followed by an equal sign, followed by zero or more characters -# representing the value that the variable is being set to. +# This will be successful only if the line consists of one or more non- +# whitespace characters representing the name of a variable followed by +# an equal sign, followed by zero or more characters representing the +# value that the variable is being set to. (Recall that in generating +# the variable default_var_defns, leading spaces on each line were +# stripped out). # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) # # If var_name is not empty, then a variable name was found on the current -# line in line_list. +# line in default_var_defns. # - if [ ! -z $var_name ]; then + if [ ! -z ${var_name} ]; then print_info_msg "$DEBUG" " var_name = \"${var_name}\"" # # If the variable specified in var_name is set in the current environment -# (to either an empty or non-empty string), get its value and insert it -# in the variable definitions file on the line where that variable is -# defined. Note that +# (to either an empty or non-empty string), get its value and save it in +# var_value. Note that # # ${!var_name+x} # # will retrun the string "x" if the variable specified in var_name is # set (to either an empty or non-empty string), and it will return an -# empty string if the variable specified in var_name is unset (i.e. un- -# defined). +# empty string if the variable specified in var_name is unset (i.e. if +# it is undefined). # - if [ ! -z ${!var_name+x} ]; then + unset "var_value" + if [ ! -z "${!var_name+x}" ]; then # # The variable may be a scalar or an array. Thus, we first treat it as # an array and obtain the number of elements that it contains. @@ -2765,49 +2701,40 @@ var_name = \"${var_name}\"" array=("${!array_name_at}") num_elems="${#array[@]}" # -# We will now set the variable var_value to the string that needs to be -# placed on the right-hand side of the assignment operator (=) on the -# appropriate line in the variable definitions file. How this is done -# depends on whether the variable is a scalar or an array. +# Set var_value to the updated value of the current experiment variable. +# How this is done depends on whether the variable is a scalar or an +# array. # # If the variable contains only one element, then it is a scalar. (It # could be a 1-element array, but for simplicity, we treat that case as # a scalar.) In this case, we enclose its value in double quotes and # save the result in var_value. # - if [ "$num_elems" -eq 1 ]; then + if [ "${num_elems}" -eq 1 ]; then + var_value="${!var_name}" - var_value="\"${var_value}\"" + rhs="'${var_value}'" # # If the variable contains more than one element, then it is an array. # In this case, we build var_value in two steps as follows: # # 1) Generate a string containing each element of the array in double -# quotes and followed by a space. +# quotes and followed by a space (and followed by an optional backslash +# and newline if multiline_arrays has been set to "TRUE"). # # 2) Place parentheses around the double-quoted list of array elements # generated in the first step. Note that there is no need to put a -# space before the closing parenthesis because in step 1, we have -# already placed a space after the last element. +# space before the closing parenthesis because during step 1 above, +# a space has already been placed after the last array element. # else - arrays_on_one_line="TRUE" - arrays_on_one_line="FALSE" - - if [ "${arrays_on_one_line}" = "TRUE" ]; then - var_value=$(printf "\"%s\" " "${!array_name_at}") -# var_value=$(printf "\"%s\" \\\\\\ \\\n" "${!array_name_at}") - else -# var_value=$(printf "%s" "\\\\\\n") - var_value="\\\\\n" - for (( i=0; i<${num_elems}; i++ )); do -# var_value=$(printf "%s\"%s\" %s" "${var_value}" "${array[$i]}" "\\\\\\n") - var_value="${var_value}\"${array[$i]}\" \\\\\n" -# var_value="${var_value}\"${array[$i]}\" " - done - fi - var_value="( $var_value)" + var_value="" + printf -v "var_value" "${escbksl_nl_or_null}" + for (( i=0; i<${num_elems}; i++ )); do + printf -v "var_value" "${var_value}\"${array[$i]}\" ${escbksl_nl_or_null}" + done + rhs="( ${var_value})" fi # @@ -2824,52 +2751,83 @@ The variable specified by \"var_name\" is not set in the current environment: var_name = \"${var_name}\" Setting its value in the variable definitions file to an empty string." - var_value="\"\"" + rhs="''" fi # -# Now place var_value on the right-hand side of the assignment statement -# on the appropriate line in the variable definitions file. +# Set the line containing the variable's definition. Then add the line +# to the list of all variable definitions. # - set_file_param "${GLOBAL_VAR_DEFNS_FP}" "${var_name}" "${var_value}" + var_defn="${var_name}=$rhs" + printf -v "var_defns" "${var_defns}${var_defn}\n" # # If var_name is empty, then a variable name was not found on the current -# line in line_list. In this case, print out a warning and move on to -# the next line. +# line in default_var_defns. In this case, print out a warning and move +# on to the next line. # else print_info_msg " -Could not extract a variable name from the current line in \"line_list\" +Could not extract a variable name from the current line in \"default_var_defns\" (probably because it does not contain an equal sign with no spaces on either side): crnt_line = \"${crnt_line}\" var_name = \"${var_name}\" -Continuing to next line in \"line_list\"." +Continuing to next line in \"default_var_defns\"." fi -done <<< "${line_list}" +done <<< "${default_var_defns}" # #----------------------------------------------------------------------- # -# Append additional variable definitions (and comments) to the variable -# definitions file. These variables have been set above using the vari- -# ables in the default and local configuration scripts. These variables -# are needed by various tasks/scripts in the workflow. +# Construct the experiment's variable definitions file. Below, we first +# record the contents we want to place in this file in the variable +# var_defns_file_contents, and we then write the contents of this +# variable to the file. # #----------------------------------------------------------------------- # -{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} +print_info_msg " +Generating the global experiment variable definitions file specified by +GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" +Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" +For more detailed information, set DEBUG to \"TRUE\" in the experiment +configuration file (\"${EXPT_CONFIG_FN}\")." +var_defns_file_contents="\ +# +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# Section 1: +# This section contains (most of) the primary experiment variables, i.e. +# those variables that are defined in the default configuration file +# (${EXPT_DEFAULT_CONFIG_FN}) and that can be reset via the user-specified +# experiment configuration file (${EXPT_CONFIG_FN}). +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# +${var_defns}" +# +# Append derived/secondary variable definitions (as well as comments) to +# the contents of the variable definitions file. +# +ensmem_names_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ENSMEM_NAMES[@]}") +ensmem_names_str=$(printf "( %s${escbksl_nl_or_null})" "${ensmem_names_str}") + +fv3_nml_ensmem_fps_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}") +fv3_nml_ensmem_fps_str=$(printf "( %s${escbksl_nl_or_null})" "${fv3_nml_ensmem_fps_str}") + +var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 2: -# This section defines variables that have been derived from the ones -# above by the setup script (setup.sh) and which are needed by one or -# more of the scripts that perform the workflow tasks (those scripts -# source this variable definitions file). +# This section defines variables that have been derived from the primary +# set of experiment variables above (we refer to these as \"derived\" or +# \"secondary\" variables). #----------------------------------------------------------------------- #----------------------------------------------------------------------- # @@ -2877,15 +2835,15 @@ done <<< "${line_list}" # #----------------------------------------------------------------------- # -# Full path to workflow launcher script, its log file, and the line that -# gets added to the cron table to launch this script if USE_CRON_TO_RELAUNCH -# is set to TRUE. +# Full path to workflow (re)launch script, its log file, and the line +# that gets added to the cron table to launch this script if the flag +# USE_CRON_TO_RELAUNCH is set to \"TRUE\". # #----------------------------------------------------------------------- # -WFLOW_LAUNCH_SCRIPT_FP="${WFLOW_LAUNCH_SCRIPT_FP}" -WFLOW_LAUNCH_LOG_FP="${WFLOW_LAUNCH_LOG_FP}" -CRONTAB_LINE="${CRONTAB_LINE}" +WFLOW_LAUNCH_SCRIPT_FP='${WFLOW_LAUNCH_SCRIPT_FP}' +WFLOW_LAUNCH_LOG_FP='${WFLOW_LAUNCH_LOG_FP}' +CRONTAB_LINE='${CRONTAB_LINE}' # #----------------------------------------------------------------------- # @@ -2893,44 +2851,44 @@ CRONTAB_LINE="${CRONTAB_LINE}" # #----------------------------------------------------------------------- # -SR_WX_APP_TOP_DIR="${SR_WX_APP_TOP_DIR}" -HOMErrfs="$HOMErrfs" -USHDIR="$USHDIR" -SCRIPTSDIR="$SCRIPTSDIR" -JOBSDIR="$JOBSDIR" -SORCDIR="$SORCDIR" -SRC_DIR="$SRC_DIR" -PARMDIR="$PARMDIR" -MODULES_DIR="${MODULES_DIR}" -EXECDIR="$EXECDIR" -FIXam="$FIXam" -FIXclim="$FIXclim" -FIXLAM="$FIXLAM" -FIXgsm="$FIXgsm" -FIXaer="$FIXaer" -FIXlut="$FIXlut" -COMROOT="$COMROOT" -COMOUT_BASEDIR="${COMOUT_BASEDIR}" -TEMPLATE_DIR="${TEMPLATE_DIR}" -VX_CONFIG_DIR="${VX_CONFIG_DIR}" -METPLUS_CONF="${METPLUS_CONF}" -MET_CONFIG="${MET_CONFIG}" -UFS_WTHR_MDL_DIR="${UFS_WTHR_MDL_DIR}" -UFS_UTILS_DIR="${UFS_UTILS_DIR}" -SFC_CLIMO_INPUT_DIR="${SFC_CLIMO_INPUT_DIR}" -TOPO_DIR="${TOPO_DIR}" -UPP_DIR="${UPP_DIR}" - -EXPTDIR="$EXPTDIR" -LOGDIR="$LOGDIR" -CYCLE_BASEDIR="${CYCLE_BASEDIR}" -GRID_DIR="${GRID_DIR}" -OROG_DIR="${OROG_DIR}" -SFC_CLIMO_DIR="${SFC_CLIMO_DIR}" - -NDIGITS_ENSMEM_NAMES="${NDIGITS_ENSMEM_NAMES}" -ENSMEM_NAMES=( $( printf "\"%s\" " "${ENSMEM_NAMES[@]}" )) -FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) +SR_WX_APP_TOP_DIR='${SR_WX_APP_TOP_DIR}' +HOMErrfs='$HOMErrfs' +USHDIR='$USHDIR' +SCRIPTSDIR='$SCRIPTSDIR' +JOBSDIR='$JOBSDIR' +SORCDIR='$SORCDIR' +SRC_DIR='${SRC_DIR}' +PARMDIR='$PARMDIR' +MODULES_DIR='${MODULES_DIR}' +EXECDIR='$EXECDIR' +FIXam='$FIXam' +FIXclim='$FIXclim' +FIXLAM='$FIXLAM' +FIXgsm='$FIXgsm' +FIXaer='$FIXaer' +FIXlut='$FIXlut' +COMROOT='$COMROOT' +COMOUT_BASEDIR='${COMOUT_BASEDIR}' +TEMPLATE_DIR='${TEMPLATE_DIR}' +VX_CONFIG_DIR='${VX_CONFIG_DIR}' +METPLUS_CONF='${METPLUS_CONF}' +MET_CONFIG='${MET_CONFIG}' +UFS_WTHR_MDL_DIR='${UFS_WTHR_MDL_DIR}' +UFS_UTILS_DIR='${UFS_UTILS_DIR}' +SFC_CLIMO_INPUT_DIR='${SFC_CLIMO_INPUT_DIR}' +TOPO_DIR='${TOPO_DIR}' +UPP_DIR='${UPP_DIR}' + +EXPTDIR='$EXPTDIR' +LOGDIR='$LOGDIR' +CYCLE_BASEDIR='${CYCLE_BASEDIR}' +GRID_DIR='${GRID_DIR}' +OROG_DIR='${OROG_DIR}' +SFC_CLIMO_DIR='${SFC_CLIMO_DIR}' + +NDIGITS_ENSMEM_NAMES='${NDIGITS_ENSMEM_NAMES}' +ENSMEM_NAMES=${ensmem_names_str} +FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} # #----------------------------------------------------------------------- # @@ -2938,46 +2896,43 @@ FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP="${GLOBAL_VAR_DEFNS_FP}" -# Try this at some point instead of hard-coding it as above; it's a more -# flexible approach (if it works). -#GLOBAL_VAR_DEFNS_FP=$( $READLINK -f "${BASH_SOURCE[0]}" ) +GLOBAL_VAR_DEFNS_FP='${GLOBAL_VAR_DEFNS_FP}' -DATA_TABLE_TMPL_FN="${DATA_TABLE_TMPL_FN}" -DIAG_TABLE_TMPL_FN="${DIAG_TABLE_TMPL_FN}" -FIELD_TABLE_TMPL_FN="${FIELD_TABLE_TMPL_FN}" -MODEL_CONFIG_TMPL_FN="${MODEL_CONFIG_TMPL_FN}" -NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_TMPL_FN}" +DATA_TABLE_TMPL_FN='${DATA_TABLE_TMPL_FN}' +DIAG_TABLE_TMPL_FN='${DIAG_TABLE_TMPL_FN}' +FIELD_TABLE_TMPL_FN='${FIELD_TABLE_TMPL_FN}' +MODEL_CONFIG_TMPL_FN='${MODEL_CONFIG_TMPL_FN}' +NEMS_CONFIG_TMPL_FN='${NEMS_CONFIG_TMPL_FN}' -DATA_TABLE_TMPL_FP="${DATA_TABLE_TMPL_FP}" -DIAG_TABLE_TMPL_FP="${DIAG_TABLE_TMPL_FP}" -FIELD_TABLE_TMPL_FP="${FIELD_TABLE_TMPL_FP}" -FV3_NML_BASE_SUITE_FP="${FV3_NML_BASE_SUITE_FP}" -FV3_NML_YAML_CONFIG_FP="${FV3_NML_YAML_CONFIG_FP}" -FV3_NML_BASE_ENS_FP="${FV3_NML_BASE_ENS_FP}" -MODEL_CONFIG_TMPL_FP="${MODEL_CONFIG_TMPL_FP}" -NEMS_CONFIG_TMPL_FP="${NEMS_CONFIG_TMPL_FP}" +DATA_TABLE_TMPL_FP='${DATA_TABLE_TMPL_FP}' +DIAG_TABLE_TMPL_FP='${DIAG_TABLE_TMPL_FP}' +FIELD_TABLE_TMPL_FP='${FIELD_TABLE_TMPL_FP}' +FV3_NML_BASE_SUITE_FP='${FV3_NML_BASE_SUITE_FP}' +FV3_NML_YAML_CONFIG_FP='${FV3_NML_YAML_CONFIG_FP}' +FV3_NML_BASE_ENS_FP='${FV3_NML_BASE_ENS_FP}' +MODEL_CONFIG_TMPL_FP='${MODEL_CONFIG_TMPL_FP}' +NEMS_CONFIG_TMPL_FP='${NEMS_CONFIG_TMPL_FP}' -CCPP_PHYS_SUITE_FN="${CCPP_PHYS_SUITE_FN}" -CCPP_PHYS_SUITE_IN_CCPP_FP="${CCPP_PHYS_SUITE_IN_CCPP_FP}" -CCPP_PHYS_SUITE_FP="${CCPP_PHYS_SUITE_FP}" +CCPP_PHYS_SUITE_FN='${CCPP_PHYS_SUITE_FN}' +CCPP_PHYS_SUITE_IN_CCPP_FP='${CCPP_PHYS_SUITE_IN_CCPP_FP}' +CCPP_PHYS_SUITE_FP='${CCPP_PHYS_SUITE_FP}' -FIELD_DICT_FN="${FIELD_DICT_FN}" -FIELD_DICT_IN_UWM_FP="${FIELD_DICT_IN_UWM_FP}" -FIELD_DICT_FP="${FIELD_DICT_FP}" +FIELD_DICT_FN='${FIELD_DICT_FN}' +FIELD_DICT_IN_UWM_FP='${FIELD_DICT_IN_UWM_FP}' +FIELD_DICT_FP='${FIELD_DICT_FP}' -DATA_TABLE_FP="${DATA_TABLE_FP}" -FIELD_TABLE_FP="${FIELD_TABLE_FP}" -FV3_NML_FN="${FV3_NML_FN}" # This may not be necessary... -FV3_NML_FP="${FV3_NML_FP}" -NEMS_CONFIG_FP="${NEMS_CONFIG_FP}" +DATA_TABLE_FP='${DATA_TABLE_FP}' +FIELD_TABLE_FP='${FIELD_TABLE_FP}' +FV3_NML_FN='${FV3_NML_FN}' +FV3_NML_FP='${FV3_NML_FP}' +NEMS_CONFIG_FP='${NEMS_CONFIG_FP}' -FV3_EXEC_FP="${FV3_EXEC_FP}" +FV3_EXEC_FP='${FV3_EXEC_FP}' -LOAD_MODULES_RUN_TASK_FP="${LOAD_MODULES_RUN_TASK_FP}" +LOAD_MODULES_RUN_TASK_FP='${LOAD_MODULES_RUN_TASK_FP}' -THOMPSON_MP_CLIMO_FN="${THOMPSON_MP_CLIMO_FN}" -THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" +THOMPSON_MP_CLIMO_FN='${THOMPSON_MP_CLIMO_FN}' +THOMPSON_MP_CLIMO_FP='${THOMPSON_MP_CLIMO_FP}' # #----------------------------------------------------------------------- # @@ -2985,115 +2940,99 @@ THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" # #----------------------------------------------------------------------- # -RELATIVE_LINK_FLAG="${RELATIVE_LINK_FLAG}" +RELATIVE_LINK_FLAG='${RELATIVE_LINK_FLAG}' # #----------------------------------------------------------------------- # # Parameters that indicate whether or not various parameterizations are -# included in and called by the phsics suite. +# included in and called by the physics suite. # #----------------------------------------------------------------------- # -SDF_USES_RUC_LSM="${SDF_USES_RUC_LSM}" -SDF_USES_THOMPSON_MP="${SDF_USES_THOMPSON_MP}" +SDF_USES_RUC_LSM='${SDF_USES_RUC_LSM}' +SDF_USES_THOMPSON_MP='${SDF_USES_THOMPSON_MP}' # #----------------------------------------------------------------------- # -# Grid configuration parameters needed regardless of grid generation me- -# thod used. +# Grid configuration parameters needed regardless of grid generation +# method used. # #----------------------------------------------------------------------- # -GTYPE="$GTYPE" -TILE_RGNL="${TILE_RGNL}" -NH0="${NH0}" -NH3="${NH3}" -NH4="${NH4}" +GTYPE='$GTYPE' +TILE_RGNL='${TILE_RGNL}' +NH0='${NH0}' +NH3='${NH3}' +NH4='${NH4}' -LON_CTR="${LON_CTR}" -LAT_CTR="${LAT_CTR}" -NX="${NX}" -NY="${NY}" -NHW="${NHW}" -STRETCH_FAC="${STRETCH_FAC}" +LON_CTR='${LON_CTR}' +LAT_CTR='${LAT_CTR}' +NX='${NX}' +NY='${NY}' +NHW='${NHW}' +STRETCH_FAC='${STRETCH_FAC}' -RES_IN_FIXLAM_FILENAMES="${RES_IN_FIXLAM_FILENAMES}" +RES_IN_FIXLAM_FILENAMES='${RES_IN_FIXLAM_FILENAMES}' # -# If running the make_grid task, CRES will be set to a null string du- -# the grid generation step. It will later be set to an actual value af- -# ter the make_grid task is complete. +# If running the make_grid task, CRES will be set to a null string during +# the grid generation step. It will later be set to an actual value after +# the make_grid task is complete. # -CRES="$CRES" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append new variable definitions to variable -definitions file returned with a nonzero status." +CRES='$CRES' +" # #----------------------------------------------------------------------- # -# Append to the variable definitions file the defintions of grid parame- -# ters that are specific to the grid generation method used. +# Append to the variable definitions file the defintions of grid parameters +# that are specific to the grid generation method used. # #----------------------------------------------------------------------- # +grid_vars_str="" if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - { cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} + grid_vars_str="\ # #----------------------------------------------------------------------- # # Grid configuration parameters for a regional grid generated from a -# global parent cubed-sphere grid. This is the method originally sug- -# gested by GFDL since it allows GFDL's nested grid generator to be used -# to generate a regional grid. However, for large regional domains, it -# results in grids that have an unacceptably large range of cell sizes +# global parent cubed-sphere grid. This is the method originally +# suggested by GFDL since it allows GFDL's nested grid generator to be +# used to generate a regional grid. However, for large regional domains, +# it results in grids that have an unacceptably large range of cell sizes # (i.e. ratio of maximum to minimum cell size is not sufficiently close # to 1). # #----------------------------------------------------------------------- # -ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append grid parameters to variable definitions -file returned with a nonzero status." +ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +" elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - { cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} + grid_vars_str="\ # #----------------------------------------------------------------------- # -# Grid configuration parameters for a regional grid generated indepen- -# dently of a global parent grid. This method was developed by Jim Pur- -# ser of EMC and results in very uniform grids (i.e. ratio of maximum to -# minimum cell size is very close to 1). +# Grid configuration parameters for a regional grid generated independently +# of a global parent grid. This method was developed by Jim Purser of +# EMC and results in very uniform grids (i.e. ratio of maximum to minimum +# cell size is very close to 1). # #----------------------------------------------------------------------- # -DEL_ANGLE_X_SG="${DEL_ANGLE_X_SG}" -DEL_ANGLE_Y_SG="${DEL_ANGLE_Y_SG}" -NEG_NX_OF_DOM_WITH_WIDE_HALO="${NEG_NX_OF_DOM_WITH_WIDE_HALO}" -NEG_NY_OF_DOM_WITH_WIDE_HALO="${NEG_NY_OF_DOM_WITH_WIDE_HALO}" -PAZI="${PAZI}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append grid parameters to variable definitions -file returned with a nonzero status." +DEL_ANGLE_X_SG='${DEL_ANGLE_X_SG}' +DEL_ANGLE_Y_SG='${DEL_ANGLE_Y_SG}' +NEG_NX_OF_DOM_WITH_WIDE_HALO='${NEG_NX_OF_DOM_WITH_WIDE_HALO}' +NEG_NY_OF_DOM_WITH_WIDE_HALO='${NEG_NY_OF_DOM_WITH_WIDE_HALO}' +PAZI='${PAZI}' +" fi -# -#----------------------------------------------------------------------- -# -# Because RUN_CMD_FCST can include PE_MEMBER01 (and theoretically other -# variables calculated in this script), delete the first occurrence of it -# in the var_defns file, and write it again at the end. -# -#----------------------------------------------------------------------- -$SED -i '/^RUN_CMD_FCST=/d' $GLOBAL_VAR_DEFNS_FP +var_defns_file_contents="${var_defns_file_contents}${grid_vars_str}" # #----------------------------------------------------------------------- # @@ -3102,15 +3041,22 @@ $SED -i '/^RUN_CMD_FCST=/d' $GLOBAL_VAR_DEFNS_FP # #----------------------------------------------------------------------- # -{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} +lbc_spec_fcst_hrs_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${LBC_SPEC_FCST_HRS[@]}") +lbc_spec_fcst_hrs_str=$(printf "( %s${escbksl_nl_or_null})" "${lbc_spec_fcst_hrs_str}") + +all_cdates_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ALL_CDATES[@]}") +all_cdates_str=$(printf "( %s${escbksl_nl_or_null})" "${all_cdates_str}") + +var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- # -# CPL: parameter for coupling in model_configure +# Flag in the \"${MODEL_CONFIG_FN}\" file for coupling the ocean model to +# the weather model. # #----------------------------------------------------------------------- # -CPL="${CPL}" +CPL='${CPL}' # #----------------------------------------------------------------------- # @@ -3119,11 +3065,11 @@ CPL="${CPL}" # #----------------------------------------------------------------------- # -OZONE_PARAM="${OZONE_PARAM}" +OZONE_PARAM='${OZONE_PARAM}' # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system +# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system # directory in which the workflow scripts will look for the files generated # by the external model specified in EXTRN_MDL_NAME_ICS. These files will # be used to generate the input initial condition and surface files for @@ -3131,11 +3077,11 @@ OZONE_PARAM="${OZONE_PARAM}" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS}" +EXTRN_MDL_SYSBASEDIR_ICS='${EXTRN_MDL_SYSBASEDIR_ICS}' # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system +# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system # directory in which the workflow scripts will look for the files generated # by the external model specified in EXTRN_MDL_NAME_LBCS. These files # will be used to generate the input lateral boundary condition files for @@ -3143,7 +3089,7 @@ EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS}" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS}" +EXTRN_MDL_SYSBASEDIR_LBCS='${EXTRN_MDL_SYSBASEDIR_LBCS}' # #----------------------------------------------------------------------- # @@ -3152,7 +3098,7 @@ EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS}" # #----------------------------------------------------------------------- # -EXTRN_MDL_LBCS_OFFSET_HRS="${EXTRN_MDL_LBCS_OFFSET_HRS}" +EXTRN_MDL_LBCS_OFFSET_HRS='${EXTRN_MDL_LBCS_OFFSET_HRS}' # #----------------------------------------------------------------------- # @@ -3161,31 +3107,33 @@ EXTRN_MDL_LBCS_OFFSET_HRS="${EXTRN_MDL_LBCS_OFFSET_HRS}" # #----------------------------------------------------------------------- # -LBC_SPEC_FCST_HRS=(${LBC_SPEC_FCST_HRS[@]}) +LBC_SPEC_FCST_HRS=${lbc_spec_fcst_hrs_str} # #----------------------------------------------------------------------- # -# The number of cycles for which to make forecasts and the list of starting -# dates/hours of these cycles. +# The number of cycles for which to make forecasts and the list of +# starting dates/hours of these cycles. # #----------------------------------------------------------------------- # -NUM_CYCLES="${NUM_CYCLES}" -ALL_CDATES=( \\ -$( printf "\"%s\" \\\\\n" "${ALL_CDATES[@]}" ) -) +NUM_CYCLES='${NUM_CYCLES}' +ALL_CDATES=${all_cdates_str} # #----------------------------------------------------------------------- # -# If USE_FVCOM is set to TRUE, then FVCOM data (located in FVCOM_DIR -# in FVCOM_FILE) will be used to update lower boundary conditions during -# make_ics. +# Parameters that determine whether FVCOM data will be used, and if so, +# their location. +# +# If USE_FVCOM is set to \"TRUE\", then FVCOM data (in the file FVCOM_FILE +# located in the directory FVCOM_DIR) will be used to update the surface +# boundary conditions during the initial conditions generation task +# (MAKE_ICS_TN). # #----------------------------------------------------------------------- # -USE_FVCOM="${USE_FVCOM}" -FVCOM_DIR="${FVCOM_DIR}" -FVCOM_FILE="${FVCOM_FILE}" +USE_FVCOM='${USE_FVCOM}' +FVCOM_DIR='${FVCOM_DIR}' +FVCOM_FILE='${FVCOM_FILE}' # #----------------------------------------------------------------------- # @@ -3193,22 +3141,26 @@ FVCOM_FILE="${FVCOM_FILE}" # #----------------------------------------------------------------------- # -NCORES_PER_NODE="${NCORES_PER_NODE}" -PE_MEMBER01="${PE_MEMBER01}" -RUN_CMD_FCST="$(eval echo \'${RUN_CMD_FCST}\')" +PE_MEMBER01='${PE_MEMBER01}' # #----------------------------------------------------------------------- # -# IF DO_SPP="TRUE," N_VAR_SPP is the number of parameterizations that -# are perturbed with SPP, otherwise N_VAR_SPP=0. +# IF DO_SPP is set to \"TRUE\", N_VAR_SPP specifies the number of physics +# parameterizations that are perturbed with SPP. Otherwise, N_VAR_SPP +# is set 0. # #----------------------------------------------------------------------- # -N_VAR_SPP="${N_VAR_SPP}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append new variable definitions to variable -definitions file returned with a nonzero status." +N_VAR_SPP='${N_VAR_SPP}' +" +# +# Done with constructing the contents of the variable definitions file, +# so now write the contents to file. +# +printf "%s\n" "${var_defns_file_contents}" >> ${GLOBAL_VAR_DEFNS_FP} + +print_info_msg "$VERBOSE" " +Done generating the global experiment variable definitions file." # #----------------------------------------------------------------------- # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 375543d354..60162aa40f 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -211,6 +211,15 @@ function source_util_funcs() { #----------------------------------------------------------------------- # . ${bashutils_dir}/create_symlink_to_file.sh +# +#----------------------------------------------------------------------- +# +# Source the file containing the function that gets the stripped contents +# of a bash script or function. +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/get_bash_file_contents.sh } source_util_funcs From 6a62056d232bc79351883e540d7a2744e874c93a Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 20 Jan 2022 12:28:08 -0500 Subject: [PATCH 114/203] Update divergence damping parameters (#657) * update nord and d4_bg * Change values of ice_nc, rain_nc, and sgs_tke for new divergence damping --- ush/load_modules_run_task.sh | 1 + ush/templates/field_table.FV3_CPT_v0 | 6 +++--- ush/templates/field_table.FV3_GFS_v15p2 | 2 +- ush/templates/field_table.FV3_GFS_v16 | 2 +- ush/templates/field_table.FV3_GSD_SAR | 6 +++--- ush/templates/field_table.FV3_GSD_v0 | 6 +++--- ush/templates/field_table.FV3_HRRR | 6 +++--- ush/templates/field_table.FV3_RRFS_v0 | 6 +++--- ush/templates/field_table.FV3_RRFS_v1alpha | 6 +++--- ush/templates/field_table.FV3_RRFS_v1beta | 6 +++--- ush/templates/input.nml.FV3 | 4 ++-- 11 files changed, 26 insertions(+), 25 deletions(-) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 17b6d5d3d4..59e3e156f8 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -136,6 +136,7 @@ module purge machine=$(echo_lowercase $MACHINE) env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" +module use "${SR_WX_APP_TOP_DIR}/env" source "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: diff --git a/ush/templates/field_table.FV3_CPT_v0 b/ush/templates/field_table.FV3_CPT_v0 index e9dabfab79..7331ae4ae4 100644 --- a/ush/templates/field_table.FV3_CPT_v0 +++ b/ush/templates/field_table.FV3_CPT_v0 @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/m3" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/m3" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic snow number concentration "TRACER", "atmos_mod", "snow_nc" "longname", "snow number concentration" @@ -63,5 +63,5 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_GFS_v15p2 b/ush/templates/field_table.FV3_GFS_v15p2 index 69cc9005f1..06adb66d72 100644 --- a/ush/templates/field_table.FV3_GFS_v15p2 +++ b/ush/templates/field_table.FV3_GFS_v15p2 @@ -34,7 +34,7 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # non-prognostic cloud amount "TRACER", "atmos_mod", "cld_amt" "longname", "cloud amount" diff --git a/ush/templates/field_table.FV3_GFS_v16 b/ush/templates/field_table.FV3_GFS_v16 index 69cc9005f1..06adb66d72 100644 --- a/ush/templates/field_table.FV3_GFS_v16 +++ b/ush/templates/field_table.FV3_GFS_v16 @@ -34,7 +34,7 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # non-prognostic cloud amount "TRACER", "atmos_mod", "cld_amt" "longname", "cloud amount" diff --git a/ush/templates/field_table.FV3_GSD_SAR b/ush/templates/field_table.FV3_GSD_SAR index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_GSD_SAR +++ b/ush/templates/field_table.FV3_GSD_SAR @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_GSD_v0 b/ush/templates/field_table.FV3_GSD_v0 index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_GSD_v0 +++ b/ush/templates/field_table.FV3_GSD_v0 @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_HRRR b/ush/templates/field_table.FV3_HRRR index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_HRRR +++ b/ush/templates/field_table.FV3_HRRR @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_RRFS_v0 b/ush/templates/field_table.FV3_RRFS_v0 index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_RRFS_v0 +++ b/ush/templates/field_table.FV3_RRFS_v0 @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_RRFS_v1alpha b/ush/templates/field_table.FV3_RRFS_v1alpha index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_RRFS_v1alpha +++ b/ush/templates/field_table.FV3_RRFS_v1alpha @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_RRFS_v1beta b/ush/templates/field_table.FV3_RRFS_v1beta index 0a927de455..fe96567e5f 100644 --- a/ush/templates/field_table.FV3_RRFS_v1beta +++ b/ush/templates/field_table.FV3_RRFS_v1beta @@ -38,12 +38,12 @@ "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic rain number concentration "TRACER", "atmos_mod", "rain_nc" "longname", "rain number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" "longname", "ozone mixing ratio" @@ -62,4 +62,4 @@ "TRACER", "atmos_mod", "sgs_tke" "longname", "subgrid scale turbulent kinetic energy" "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 645e216011..450b1361a9 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -60,7 +60,7 @@ d2_bg = 0.0 d2_bg_k1 = 0.20 d2_bg_k2 = 0.04 - d4_bg = 0.15 + d4_bg = 0.12 d_con = 1.0 d_ext = 0.0 dddmp = 0.1 @@ -100,7 +100,7 @@ ncep_ic = .false. nggps_ic = .true. no_dycore = .false. - nord = 2 + nord = 3 npz = 64 nrows_blend = 10 ntiles = 1 From eb435aa25037f1cec04a3dc93860ffb870eb1df6 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:33:36 -0600 Subject: [PATCH 115/203] Add stand-alone Python data ingest tool. (#667) * Adding YAML file with fixed GFS data locations. * Use "download" as protocol. * Adding WIP script to retrieve data using yml. Does not work at all at this point. * Updates to allow working ingest. * Turn on logging and clean up after htar. * Adding support for RAP/HRRR/NAM * WIP. * Fixing paths, testing HRRR, error if nothing found. * Better handling of un-found dataset via wget. * Adding capability to copy files from disk. * Linting. * Addressing review concerns. --- ush/retrieve_data.py | 707 +++++++++++++++++++++++++++++++ ush/templates/data_locations.yml | 193 +++++++++ 2 files changed, 900 insertions(+) create mode 100644 ush/retrieve_data.py create mode 100644 ush/templates/data_locations.yml diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py new file mode 100644 index 0000000000..48210c0d53 --- /dev/null +++ b/ush/retrieve_data.py @@ -0,0 +1,707 @@ +# pylint: disable=logging-fstring-interpolation +''' +This script helps users pull data from known data streams, including +URLS and HPSS (only on supported NOAA platforms), or from user-supplied +data locations on disk. + +Several supported data streams are included in +ush/templates/data_locations.yml, which provides locations and naming +conventions for files commonly used with the SRW App. Provide the file +to this tool via the --config flag. Users are welcome to provide their +own file with alternative locations and naming conventions. + +When using this script to pull from disk, the user is required to +provide the path to the data location, which can include Python +templates. The file names follow those included in the --config file by +default, or can be user-supplied via the --file_name flag. That flag +takes a YAML-formatted string that follows the same conventions outlined +in the ush/templates/data_locations.yml file for naming files. + +To see usage for this script: + + python retrieve_data.py -h + +Also see the parse_args function below. +''' + +import argparse +import datetime as dt +import logging +import os +import shutil +import subprocess +import sys + +import yaml + +def clean_up_output_dir(expected_subdir, local_archive, output_path, source_paths): + + ''' Remove expected sub-directories and existing_archive files on + disk once all files have been extracted and put into the specified + output location. ''' + + unavailable = {} + # Check to make sure the files exist on disk + for file_path in source_paths: + local_file_path = os.path.join(output_path,file_path) + if not os.path.exists(local_file_path): + logging.info(f'File does not exist: {local_file_path}') + unavailable['hpss'] = source_paths + else: + file_name = os.path.basename(file_path) + expected_output_loc = os.path.join(output_path, file_name) + if not local_file_path == expected_output_loc: + logging.info(f'Moving {local_file_path} to ' \ + f'{expected_output_loc}') + shutil.move(local_file_path, expected_output_loc) + + # Clean up directories from inside archive, if they exist + if os.path.exists(expected_subdir) and expected_subdir != './': + logging.info(f'Removing {expected_subdir}') + os.removedirs(expected_subdir) + + # If an archive exists on disk, remove it + if os.path.exists(local_archive): + os.remove(local_archive) + + return unavailable + +def copy_file(source, destination): + + ''' + Copy a file from a source and place it in the destination location. + Return a boolean value reflecting the state of the copy. + + Assumes destination exists. + ''' + + if not os.path.exists(source): + logging.info(f'File does not exist on disk \n {source}') + return False + + # Using subprocess here because system copy is much faster than + # python copy options. + cmd = f'cp {source} {destination}' + logging.info(f'Running command: \n {cmd}') + try: + subprocess.run(cmd, + check=True, + shell=True, + ) + except subprocess.CalledProcessError as err: + logging.info(err) + return False + return True + +def download_file(url): + + ''' + Download a file from a url source, and place it in a target location + on disk. + + Arguments: + url url to file to be downloaded + + Return: + boolean value reflecting state of download. + ''' + + # wget flags: + # -c continue previous attempt + # -T timeout seconds + # -t number of tries + cmd = f'wget -c -T 30 -t 3 {url}' + logging.info(f'Running command: \n {cmd}') + try: + subprocess.run(cmd, + check=True, + shell=True, + ) + except subprocess.CalledProcessError as err: + logging.info(err) + return False + except: + logging.error('Command failed!') + raise + + return True + +def download_requested_files(cla, data_store, store_specs): + + ''' This function interacts with the "download" protocol in a + provided data store specs file to download a set of files requested + by the user. It calls download_file for each individual file that + should be downloaded. ''' + + base_urls = store_specs['url'] + base_urls = base_urls if isinstance(base_urls, list) else [base_urls] + + file_names = store_specs.get('file_names', {}) + if cla.file_type is not None: + file_names = file_names[cla.file_type] + file_names = file_names[cla.anl_or_fcst] + target_path = fill_template(cla.output_path, + cla.cycle_date) + + logging.info(f'Downloaded files will be placed here: \n {target_path}') + orig_path = os.getcwd() + os.chdir(target_path) + unavailable = {} + for base_url in base_urls: + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + url = os.path.join(base_url, file_name) + url = fill_template(url, cla.cycle_date, fcst_hr) + downloaded = download_file(url) + if not downloaded: + + if unavailable.get(data_store) is None: + unavailable[data_store] = [] + unavailable[data_store].append(target_path) + os.chdir(orig_path) + # Returning here assumes that if the first file + # isn't found, none of the others will be. Don't + # waste time timing out on every requested file. + return unavailable + os.chdir(orig_path) + return unavailable + +def fhr_list(args): + + ''' + Given an argparse list argument, return the sequence of forecast hours to + process. + + The length of the list will determine what forecast hours are returned: + + Length = 1: A single fhr is to be processed + Length = 2: A sequence of start, stop with increment 1 + Length = 3: A sequence of start, stop, increment + Length > 3: List as is + + argparse should provide a list of at least one item (nargs='+'). + + Must ensure that the list contains integers. + ''' + + args = args if isinstance(args, list) else [args] + arg_len = len(args) + if arg_len in (2, 3): + args[1] += 1 + return list(range(*args)) + + return args + +def fill_template(template_str, cycle_date, fcst_hr=0, + templates_only=False): + + ''' Fill in the provided template string with date time information, + and return the resulting string. + + Arguments: + template_str a string containing Python templates + cycle_date a datetime object that will be used to fill in + date and time information + fcst_hr an integer forecast hour. string formatting should + be included in the template_str + templates_only boolean value. When True, this function will only + return the templates available. + + Rerturn: + filled template string + ''' + + cycle_hour = cycle_date.strftime('%H') + # One strategy for binning data files at NCEP is to put them into 6 + # cycle bins. The archive file names include the low and high end of the + # range. Set the range as would be indicated in the archive file + # here. Integer division is intentional here. + low_end = int(cycle_hour) // 6 * 6 + bin6 = f'{low_end:02d}-{low_end+5:02d}' + + # Another strategy is to bundle odd cycle hours with their next + # lowest even cycle hour. Files are named only with the even hour. + # Integer division is intentional here. + hh_even = f'{int(cycle_hour) // 2 * 2:02d}' + + format_values = dict( + bin6=bin6, + fcst_hr=fcst_hr, + dd=cycle_date.strftime('%d'), + hh=cycle_hour, + hh_even=hh_even, + jjj=cycle_date.strftime('%j'), + mm=cycle_date.strftime('%m'), + yy=cycle_date.strftime('%y'), + yyyy=cycle_date.strftime('%Y'), + yyyymm=cycle_date.strftime('%Y%m'), + yyyymmdd=cycle_date.strftime('%Y%m%d'), + yyyymmddhh=cycle_date.strftime('%Y%m%d%H'), + ) + if templates_only: + return f'{",".join((format_values.keys()))}' + return template_str.format(**format_values) + +def find_archive_files(paths, file_names, cycle_date): + + ''' Given an equal-length set of archive paths and archive file + names, and a cycle date, check HPSS via hsi to make sure at least + one set exists. Return the path of the existing archive, along with + the item in set of paths that was found.''' + + zipped_archive_file_paths = zip(paths, file_names) + + # Narrow down which HPSS files are available for this date + for list_item, (archive_path, archive_file_names) in \ + enumerate(zipped_archive_file_paths): + + if not isinstance(archive_file_names, list): + archive_file_names = [archive_file_names] + + # Only test the first item in the list, it will tell us if this + # set exists at this date. + file_path = os.path.join(archive_path, archive_file_names[0]) + file_path = fill_template(file_path, cycle_date) + + existing_archive = hsi_single_file(file_path) + + if existing_archive: + logging.info(f'Found HPSS file: {file_path}') + return existing_archive, list_item + + return '', 0 + +def get_requested_files(cla, file_names, input_loc, method='disk'): + + ''' This function copies files from disk locations + or downloads files from a url, depending on the option specified for + user. + + This function expects that the output directory exists and is + writeable. + + Arguments: + + cla Namespace object containing command line arguments + file_names Dict of file names by file type and kind + input_loc A string containing a single data location, either a url + or disk path. + method Choice of disk or download to indicate protocol for + retrieval + + Returns + unavailable a dict whose keys are "method" and whose values are a + list of files unretrievable + ''' + + unavailable = {} + + if cla.file_type is not None: + file_names = file_names[cla.file_type] + file_names = file_names[cla.anl_or_fcst] + + file_names = file_names if isinstance(file_names, list) else \ + [file_names] + target_path = fill_template(cla.output_path, + cla.cycle_date) + + logging.info(f'Retrieved files will be placed here: \n {target_path}') + orig_path = os.getcwd() + os.chdir(target_path) + unavailable = {} + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + loc = os.path.join(input_loc, file_name) + loc = fill_template(loc, cla.cycle_date, fcst_hr) + + if method == 'disk': + retrieved = copy_file(loc, target_path) + + if method == 'download': + retrieved = download_file(loc) + + if not retrieved: + + if unavailable.get(method) is None: + unavailable[method] = [] + unavailable[method].append(target_path) + os.chdir(orig_path) + # Returning here assumes that if the first file + # isn't found, none of the others will be. Don't + # waste time timing out on every requested file. + return unavailable + os.chdir(orig_path) + return unavailable + +def hsi_single_file(file_path, mode='ls'): + + ''' Call hsi as a subprocess for Python and return information about + whether the file_path was found. + + Arguments: + file_path path on HPSS + mode the hsi command to run. ls is default. may also + pass "get" to retrieve the file path + + ''' + cmd = f'hsi {mode} {file_path}' + + logging.info(f'Running command \n {cmd}') + try: + subprocess.run(cmd, + check=True, + shell=True, + ) + except subprocess.CalledProcessError: + logging.warning(f'{file_path} is not available!') + return '' + + return file_path + +def hpss_requested_files(cla, store_specs): + + ''' This function interacts with the "hpss" protocol in a + provided data store specs file to download a set of files requested + by the user. Depending on the type of archive file (zip or tar), it + will either pull the entire file and unzip it, or attempt to pull + individual files from a tar file. + + It cleans up local disk after files are deemed available to remove + any empty subdirectories that may still be present. + + This function exepcts that the output directory exists and is + writable. + ''' + + archive_paths = store_specs['archive_path'] + archive_paths = archive_paths if isinstance(archive_paths, list) \ + else [archive_paths] + + # Could be a list of lists + archive_file_names = store_specs.get('archive_file_names', {}) + if cla.file_type is not None: + archive_file_names = archive_file_names[cla.file_type] + + if isinstance(archive_file_names, dict): + archive_file_names = archive_file_names[cla.anl_or_fcst] + + unavailable = {} + existing_archive = None + + logging.debug(f'Will try to look for: '\ + f' {list(zip(archive_paths, archive_file_names))}') + + existing_archive, which_archive = find_archive_files(archive_paths, + archive_file_names, + cla.cycle_date, + ) + + if not existing_archive: + logging.warning('No archive files were found!') + unavailable['archive'] = list(zip(archive_paths, archive_file_names)) + return unavailable + + # Use the found archive file path to get the necessary files + file_names = store_specs.get('file_names', {}) + if cla.file_type is not None: + file_names = file_names[cla.file_type] + file_names = file_names[cla.anl_or_fcst] + + logging.debug(f'Grabbing archive number {which_archive} in list.') + archive_internal_dir = store_specs.get('archive_internal_dir', [''])[which_archive] + archive_internal_dir = fill_template(archive_internal_dir, + cla.cycle_date) + + output_path = fill_template(cla.output_path, cla.cycle_date) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + orig_path = os.getcwd() + os.chdir(output_path) + logging.debug(f'CWD: {os.getcwd()}') + + source_paths = [] + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + source_paths.append(fill_template( + os.path.join(archive_internal_dir, file_name), + cla.cycle_date, + fcst_hr, + )) + + if store_specs.get('archive_format', 'tar') == 'zip': + # Get the entire file from HPSS + existing_archive = hsi_single_file(existing_archive, mode='get') + + # Grab only the necessary files from the archive + cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + + else: + cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) + + # Check that files exist and Remove any data transfer artifacts. + unavailable = clean_up_output_dir( + expected_subdir=archive_internal_dir, + local_archive=os.path.basename(existing_archive), + output_path=output_path, + source_paths=source_paths, + ) + + os.chdir(orig_path) + + return unavailable + +def load_str(arg): + + ''' Load a dict string safely using YAML. Return the resulting dict. ''' + return yaml.load(arg, Loader=yaml.SafeLoader) + +def config_exists(arg): + + ''' + Check to ensure that the provided config file exists. If it does, load it + with YAML's safe loader and return the resulting dict. + ''' + + # Check for existence of file + if not os.path.exists(arg): + msg = f'{arg} does not exist!' + raise argparse.ArgumentTypeError(msg) + + with open(arg, 'r') as config_path: + cfg = yaml.load(config_path, Loader=yaml.SafeLoader) + return cfg + +def path_exists(arg): + + ''' Check whether the supplied path exists and is writeable ''' + + if not os.path.exists(arg): + msg = f'{arg} does not exist!' + raise argparse.ArgumentTypeError(msg) + + if not os.access(arg, os.X_OK|os.W_OK): + logging.error(f'{arg} is not writeable!') + raise argparse.ArgumentTypeError(msg) + + return arg + +def setup_logging(debug=False): + + ''' Calls initialization functions for logging package, and sets the + user-defined level for logging in the script.''' + + level = logging.WARNING + if debug: + level = logging.DEBUG + + logging.basicConfig(format='%(levelname)s: %(message)s \n ', level=level) + if debug: + logging.info('Logging level set to DEBUG') + + + +def to_datetime(arg): + ''' Return a datetime object give a string like YYYYMMDDHH. + ''' + + return dt.datetime.strptime(arg, '%Y%m%d%H') + +def to_lower(arg): + ''' Return a string provided by arg into all lower case. ''' + return arg.lower() + +def main(cla): + ''' + Uses known location information to try the known locations and file + paths in priority order. + ''' + + setup_logging(cla.debug) + + known_data_info = cla.config.get(cla.external_model) + if known_data_info is None: + msg = ('No data stores have been defined for', + f'{cla.external_model}!') + raise KeyError(msg) + + unavailable = {} + for data_store in cla.data_stores: + logging.info(f'Checking {data_store} for {cla.external_model}') + store_specs = known_data_info.get(data_store, {}) + + if data_store == 'disk': + file_names = cla.file_names if cla.file_names else \ + known_data_info.get('hpss', {}).get('file_names') + logging.debug(f'User supplied file names are: {file_names}') + if not file_names: + msg = ('No file name found. They must be provided \ + either on the command line or on in a config file.') + raise argparse.ArgumentTypeError(msg) + unavailable = get_requested_files(cla, + file_names=file_names, + input_loc=cla.input_file_path, + method='disk', + ) + + elif not store_specs: + msg = (f'No information is available for {data_store}.') + raise KeyError(msg) + + if store_specs.get('protocol') == 'download': + file_names = store_specs.get('file_names') + if not file_names: + msg = ('No file name found. They must be provided \ + either on the command line or on in a config file.') + raise argparse.ArgumentTypeError(msg) + + unavailable = get_requested_files(cla, + file_names=file_names, + input_loc=store_specs['url'], + method='download', + ) + + if store_specs.get('protocol') == 'htar': + unavailable = hpss_requested_files(cla, store_specs) + + if not unavailable: + # All files are found. Stop looking! + break + + logging.warning(f'Requested files are unavialable from {data_store}') + + if unavailable: + logging.error('Could not find any of the requested files.') + sys.exit(1) + +def parse_args(): + + ''' + Function maintains the arguments accepted by this script. Please see + Python's argparse documenation for more information about settings of each + argument. + ''' + + description=( + 'Allowable Python templates for paths, urls, and file names are '\ + ' defined in the fill_template function and include:\n' \ + f'{"-"*120}\n' \ + f'{fill_template("null", dt.datetime.now(), templates_only=True)}') + parser = argparse.ArgumentParser( + description=description, + ) + + # Required + parser.add_argument( + '--anl_or_fcst', + choices=('anl', 'fcst'), + help='Flag for whether analysis or forecast \ + files should be gathered', + required=True, + ) + parser.add_argument( + '--config', + help='Full path to a configuration file containing paths and \ + naming conventions for known data streams. The default included \ + in this repository is in ush/templates/data_locations.yml', + type=config_exists, + ) + parser.add_argument( + '--cycle_date', + help='Cycle date of the data to be retrieved in YYYYMMDDHH \ + format.', + required=True, + type=to_datetime, + ) + parser.add_argument( + '--data_stores', + help='List of priority data_stores. Tries first list item \ + first. Choices: hpss, nomads, aws, disk', + nargs='*', + required=True, + type=to_lower, + ) + parser.add_argument( + '--external_model', + choices=('FV3GFS', 'GSMGFS', 'HRRR', 'NAM', 'RAP', 'RAPx', + 'HRRRx'), + help='External model label. This input is case-sensitive', + required=True, + ) + parser.add_argument( + '--fcst_hrs', + help='A list describing forecast hours. If one argument, \ + one fhr will be processed. If 2 or 3 arguments, a sequence \ + of forecast hours [start, stop, [increment]] will be \ + processed. If more than 3 arguments, the list is processed \ + as-is.', + nargs='+', + required=True, + type=int, + ) + parser.add_argument( + '--output_path', + help='Path to a location on disk. Path is expected to exist.', + required=True, + type=os.path.abspath, + ) + + # Optional + parser.add_argument( + '--debug', + action='store_true', + help='Print debug messages', + ) + parser.add_argument( + '--file_names', + help='A YAML-formatted string that indicates the naming \ + convention the be used for the files retrieved from disk. If \ + not provided, the default names from hpss are used.', + type=load_str, + ) + parser.add_argument( + '--file_type', + choices=('grib2', 'nemsio', 'netcdf'), + help='External model file format', + ) + parser.add_argument( + '--input_file_path', + help='A path to data stored on disk. The path may contain \ + Python templates. File names may be supplied using the \ + --file_names flag, or the default naming convention will be \ + taken from the --config file.', + nargs='*', + ) + return parser.parse_args() + +if __name__ == '__main__': + + CLA = parse_args() + CLA.output_path = path_exists(CLA.output_path) + CLA.fcst_hrs = fhr_list(CLA.fcst_hrs) + + if 'disk' in CLA.data_stores: + # Make sure a path was provided. + if not CLA.input_file_path: + raise argparse.ArgumentTypeError( + ('You must provide an input_file_path when choosing ' \ + ' disk as a data store!')) + + if 'hpss' in CLA.data_stores: + # Make sure hpss module is loaded + try: + output = subprocess.run('which hsi', + check=True, + shell=True, + capture_output=True, + ) + except subprocess.CalledProcessError: + logging.error('You requested the hpss data store, but ' \ + 'the HPSS module isn\'t loaded. This data store ' \ + 'is only available on NOAA compute platforms.') + + main(CLA) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml new file mode 100644 index 0000000000..23354e1719 --- /dev/null +++ b/ush/templates/data_locations.yml @@ -0,0 +1,193 @@ +# This file tracks known data store locations for data used as input to +# the SRW App configuations. +# +# The contents should be organized as follows: +# +# Top level: A key corresponding to an external model file or +# observation dataset accepted by the SRW App. +# +# 2nd level: A key corressponding to a named datastore for a given data +# set. This could be nomads, aws, hpss, etc. +# +# 3rd level required: +# +# protocol: +# indication of how a set of files will be retrieved. Options: +# download or htar. The requirements for each protocol are described +# below +# +# file_names: +# Each data store will require an entry that describes the names of +# files to be extracted from the data store. These entries are +# expected as lists to indicate when multiple files should be +# retrieved. anl and fcst subsections indicate the naming convention +# for that type of file. Both are required for external models ICS and +# LBCS. Each file name may (and likely should) include templates. +# +# If the SRW App accepts different file formats, +# as it does for FV3GFS, an intermediary level indicating the value +# associate with a FV3GFS_FILE_FMT_* variable. +# +# 3rd level optional: +# +# for download protocol: +# url: required. the URL to the location of the data file. May include +# templates. +# +# for htar protocol: +# archive_path: a list of paths to the potential location of the +# archive file on HPSS. since there may be multiple +# options for a given external model file, the list +# will be searched in order listed below. +# archive_file_names: the name of the archive file. this could +# differ for each archive_path option, so one entry is +# needed (even if it's identical) for each entry of +# archive_path provided +# archive_internal_dir: (optional) a path to data files stored +# inside a given archive file +# +# +# +FV3GFS: + nomads: + protocol: download + url: https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.{yyyymmdd}/{hh}/atmos + file_names: &gfs_file_names + grib2: + anl: + - gfs.t{hh}z.pgrb2.0p25.f000 + fcst: + - gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} + nemsio: + anl: + - gfs.t{hh}z.atmanl.nemsio + - gfs.t{hh}z.sfcanl.nemsio + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio + - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio + hpss: + protocol: htar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + - ./gfs.{yyyymmdd}/{hh} + - ./gfs.{yyyymmdd}/{hh}/atmos + archive_file_names: + grib2: + anl: + - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar + - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar + fcst: + - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar + - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar + nemsio: + anl: + - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsioa.tar + - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsioa.tar + fcst: + - ['gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsioa.tar', 'gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsiob.tar'] + - ['com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsioa.tar', 'com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nemsiob.tar'] + netcdf: + anl: + - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar + - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar + fcst: + - ['gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar'] + - ['com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar'] + file_names: + <<: *gfs_file_names + aws: + protocol: download + url: https://noaa-gfs-bdp-pds.s3.amazonaws.com/gfs.{yyyymmdd}/{hh}/atmos + file_names: + <<: *gfs_file_names + +RAP: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + - ./ + archive_file_names: + # RAP forecasts are binned into 6 hour tar files. + - com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar + file_names: &rap_file_names + anl: + - rap.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + fcst: + - rap.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + aws: + protocol: download + url: https://noaa-rap-pds.s3.amazonaws.com/rap.{yyyymmdd} + file_names: + <<: *rap_file_names + +RAPx: + hpss: + protocol: htar + archive_format: zip + archive_path: + - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/rap/full/wrfnat + archive_file_names: + # RAPx bins two cycles togehter, and named by the lower even value + # of the cycle hour. + - '{yyyymmdd}{hh_even}00.zip' + file_names: + anl: + - '{yy}{jjj}{hh}00{fcst_hr:02d}00' + fcst: + - '{yy}{jjj}{hh}00{fcst_hr:02d}00' + +HRRR: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + - ./ + archive_file_names: + # HRRR forecasts are binned into 6 hour tar files. + - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrfnatdng.tar + file_names: &hrrr_file_names + anl: + - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + fcst: + - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + aws: + protocol: download + url: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.{yyyymmdd}/conus/ + file_names: + <<: *hrrr_file_names + +HRRRx: + hpss: + protocol: htar + archive_format: zip + archive_path: + - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/hrrr/conus/wrfnat + archive_file_names: + - '{yyyymmddhh}00.zip' + file_names: + anl: + - '{yy}{jjj}{hh}00{fcst_hr:02d}00' + fcst: + - '{yy}{jjj}{hh}00{fcst_hr:02d}00' + +NAM: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_file_names: + - com_nam_prod_nam.{yyyymmddhh}.bgrid.tar + file_names: + anl: + - nam.t{hh}z.bgrdsf{fcst_hr:03d}.tm00 + fcst: + - nam.t{hh}z.bgrdsf{fcst_hr:03d} + From 80101f7e338c90e76dc074352cc711da77127bab Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Mon, 31 Jan 2022 17:52:41 -0600 Subject: [PATCH 116/203] Fixing path to input data on Jet. (#669) --- tests/WE2E/run_WE2E_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index dd1f506133..15db83c3d3 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -938,7 +938,7 @@ PTMP=\"${PTMP}\"" elif [ "$MACHINE" = "HERA" ]; then extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" elif [ "$MACHINE" = "JET" ]; then - extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" + extrn_mdl_source_basedir="/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" elif [ "$MACHINE" = "CHEYENNE" ]; then extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" elif [ "$MACHINE" = "ORION" ]; then From 2e2523ba8d2c2765a379bc7b36b4f4cc2fc1735a Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:33:59 -0600 Subject: [PATCH 117/203] Make platform information configurable. (#666) ## DESCRIPTION OF CHANGES: These changes move the machine-specific configuration information from the configuration, run time, and test scripts (at least a huge portion of it, with some exceptions for run time environment management on WCOSS machines). The machine-specific information now lives in `ush/machine/*.sh` files. In addition, a new bash_util, boolify, was added to clean up `ush/setup.sh`. The function can be called for any boolean flag that can be provided with valid values like `("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")`. --- scripts/exregional_make_grid.sh | 78 +- scripts/exregional_make_ics.sh | 74 +- scripts/exregional_make_lbcs.sh | 73 +- scripts/exregional_make_orog.sh | 70 +- scripts/exregional_make_sfc_climo.sh | 57 +- scripts/exregional_run_fcst.sh | 70 +- scripts/exregional_run_post.sh | 58 +- tests/WE2E/run_WE2E_tests.sh | 231 +++--- ush/bash_utils/boolify.sh | 69 ++ ush/config_defaults.sh | 20 +- ush/get_extrn_mdl_file_dir_info.sh | 50 +- ush/machine/cheyenne.sh | 76 ++ ush/machine/hera.sh | 81 ++ ush/machine/jet.sh | 86 +++ ush/machine/odin.sh | 64 ++ ush/machine/orion.sh | 67 ++ ush/machine/stampede.sh | 64 ++ ush/machine/wcoss_cray.sh | 77 ++ ush/machine/wcoss_dell_p3.sh | 78 ++ ush/make_grid_mosaic_file.sh | 3 +- ush/set_extrn_mdl_params.sh | 109 +-- ush/setup.sh | 1017 ++++++-------------------- ush/source_util_funcs.sh | 9 + 23 files changed, 1154 insertions(+), 1427 deletions(-) create mode 100644 ush/bash_utils/boolify.sh create mode 100755 ush/machine/cheyenne.sh create mode 100755 ush/machine/hera.sh create mode 100755 ush/machine/jet.sh create mode 100755 ush/machine/odin.sh create mode 100755 ush/machine/orion.sh create mode 100755 ush/machine/stampede.sh create mode 100755 ush/machine/wcoss_cray.sh create mode 100755 ush/machine/wcoss_dell_p3.sh diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 2b876401dc..ebc580a4da 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -92,7 +92,7 @@ case "$MACHINE" in module list { restore_shell_opts; } > /dev/null 2>&1 export NODES=1 - export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" + export RUN_CMD_SERIAL="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" export KMP_AFFINITY=disabled ulimit -s unlimited ulimit -a @@ -102,56 +102,25 @@ case "$MACHINE" in { save_shell_opts; set +x; } > /dev/null 2>&1 module list { restore_shell_opts; } > /dev/null 2>&1 - export APRUN="mpirun" + export RUN_CMD_SERIAL="mpirun" ulimit -s unlimited ;; - "HERA") - APRUN="time" - ;; - - "ORION") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; - - "JET") - APRUN="time" - ulimit -a - ;; - - "ODIN") - export APRUN="srun -n 1" - ulimit -s unlimited - ulimit -a - ;; - - "CHEYENNE") - APRUN="time" - ;; - - "STAMPEDE") - export APRUN="time" - ulimit -s unlimited - ulimit -a - ;; - - "MACOS") - APRUN=time - ;; - - "LINUX") - APRUN=time - ;; - *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +if [ -z ${RUN_CMD_SERIAL:-} ] ; then + print_err_msg_exit " \ + Run command was not set in machine file. \ + Please set RUN_CMD_SERIAL for your platform" +else + RUN_CMD_SERIAL=$(eval echo ${RUN_CMD_SERIAL}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." +fi # #----------------------------------------------------------------------- # @@ -311,7 +280,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # for the 6 global tiles. However, after this call we will only need the # regional grid file. # - $APRUN ${exec_fp} \ + $RUN_CMD_SERIAL ${exec_fp} \ --grid_type gnomonic_ed \ --nlon ${nx_t6sg} \ --grid_name ${grid_name} \ @@ -385,7 +354,7 @@ $settings" # # Call the executable that generates the grid file. # - $APRUN ${exec_fp} ${rgnl_grid_nml_fp} || \ + $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates a ESGgrid-type regional grid returned with nonzero exit code: @@ -424,7 +393,7 @@ cubed-sphere grid equivalent resolution does not exist: Please ensure that you've built this executable." fi -$APRUN ${exec_fp} "${grid_fp}" || \ +$RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that calculates the regional grid's global uniform cubed-sphere grid equivalent resolution returned with nonzero exit @@ -519,7 +488,7 @@ fi exec_fn="shave" exec_fp="$EXECDIR/${exec_fn}" if [ ! -f "${exec_fp}" ]; then - print_err_msg_exit "\ + print_err_msg_exit " \ The executable (exec_fp) for \"shaving\" down the halo in the grid file does not exist: exec_fp = \"${exec_fp}\" @@ -553,7 +522,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH3} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$APRUN ${exec_fp} < ${nml_fn} || \ +$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH3}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero @@ -579,7 +548,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$APRUN ${exec_fp} < ${nml_fn} || \ +$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH4}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero @@ -603,7 +572,8 @@ cd_vrfy - make_grid_mosaic_file \ grid_dir="${GRID_DIR}" \ grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NHW}.nc" \ - mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" || \ + mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" \ + run_cmd="${RUN_CMD_SERIAL}" || \ print_err_msg_exit "\ Call to function to generate the mosaic file for a grid with a ${NHW}-cell-wide halo failed." @@ -617,7 +587,8 @@ halo failed." make_grid_mosaic_file \ grid_dir="${GRID_DIR}" \ grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" \ - mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" || \ + mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" \ + run_cmd="${RUN_CMD_SERIAL}" || \ print_err_msg_exit "\ Call to function to generate the mosaic file for a grid with a ${NH3}-cell-wide halo failed." @@ -631,7 +602,8 @@ halo failed." make_grid_mosaic_file \ grid_dir="${GRID_DIR}" \ grid_fn="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" \ - mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" || \ + mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" \ + run_cmd="${RUN_CMD_SERIAL}" || \ print_err_msg_exit "\ Call to function to generate the mosaic file for a grid with a ${NH4}-cell-wide halo failed." diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 67915c6095..7161f16aab 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -90,59 +90,33 @@ case "$MACHINE" in "WCOSS_CRAY") ulimit -s unlimited - APRUN="aprun -b -j1 -n48 -N12 -d1 -cc depth" + RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth" ;; "WCOSS_DELL_P3") ulimit -s unlimited - APRUN="mpirun" + RUN_CMD_UTILS="mpirun" ;; - "HERA") - ulimit -s unlimited - APRUN="srun" - ;; - - "ORION") - ulimit -s unlimited - ulimit -a - APRUN="srun" - ;; - - "JET") - ulimit -s unlimited - APRUN="srun" - ;; - - "ODIN") - APRUN="srun" - ;; - - "CHEYENNE") - nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) - APRUN="mpirun -np $nprocs" + *) + source ${MACHINE_FILE} ;; - "STAMPEDE") - APRUN="ibrun" - ;; +esac - "MACOS") - APRUN=$RUN_CMD_UTILS - ;; +nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) - "LINUX") - ulimit -s unlimited - APRUN=$RUN_CMD_UTILS - ;; +if [ -z ${RUN_CMD_UTILS:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_UTILS for your platform" +else + RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_UTILS}\'." +fi - *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" - ;; -esac # #----------------------------------------------------------------------- # @@ -618,14 +592,14 @@ $settings" #----------------------------------------------------------------------- # # NOTE: -# Often when the chgres_cube.exe run fails, it still returns a zero re- -# turn code, so the failure isn't picked up the the logical OR (||) be- -# low. That should be fixed. This might be due to the APRUN command - -# maybe that is returning a zero exit code even though the exit code -# of chgres_cube is nonzero. -# A similar thing happens in the forecast task. -# -${APRUN} ${exec_fp} || \ +# Often when the chgres_cube.exe run fails, it still returns a zero +# return code, so the failure isn't picked up the the logical OR (||) +# below. That should be fixed. This might be due to the RUN_CMD_UTILS +# command - maybe that is returning a zero exit code even though the +# exit code of chgres_cube is nonzero. A similar thing happens in the +# forecast task. +# +${RUN_CMD_UTILS} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions (ICs) files for the FV3-LAM failed: @@ -685,7 +659,7 @@ Please check the following user defined variables: cp_vrfy ${fvcom_data_fp} ${ics_dir}/fvcom.nc cd_vrfy ${ics_dir} - ${APRUN} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \ + ${RUN_CMD_UTILS} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \ print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index c28fe24ebb..f123dfc408 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -90,58 +90,31 @@ case "$MACHINE" in "WCOSS_CRAY") ulimit -s unlimited - APRUN="aprun -b -j1 -n48 -N12 -d1 -cc depth" + RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth" ;; "WCOSS_DELL_P3") ulimit -s unlimited - APRUN="mpirun" - ;; - - "HERA") - ulimit -s unlimited - APRUN="srun" - ;; - - "ORION") - ulimit -s unlimited - APRUN="srun" - ;; - - "JET") - ulimit -s unlimited - APRUN="srun" - ;; - - "ODIN") - APRUN="srun" - ;; - - "CHEYENNE") - nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS )) - APRUN="mpirun -np $nprocs" - ;; - - "STAMPEDE") - APRUN="ibrun" - ;; - - "MACOS") - APRUN=$RUN_CMD_UTILS - ;; - - "LINUX") - ulimit -s unlimited - APRUN=$RUN_CMD_UTILS + RUN_CMD_UTILS="mpirun" ;; *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS )) + +if [ -z ${RUN_CMD_UTILS:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_UTILS for your platform" +else + RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_UTILS}\'." +fi # #----------------------------------------------------------------------- # @@ -532,14 +505,14 @@ $settings" #----------------------------------------------------------------------- # # NOTE: -# Often when the chgres_cube.exe run fails, it still returns a zero re- -# turn code, so the failure isn't picked up the the logical OR (||) be- -# low. That should be fixed. This might be due to the APRUN command - -# maybe that is returning a zero exit code even though the exit code -# of chgres_cube is nonzero. -# A similar thing happens in the forecast task. -# - ${APRUN} ${exec_fp} || \ +# Often when the chgres_cube.exe run fails, it still returns a zero +# return code, so the failure isn't picked up the the logical OR (||) +# below. That should be fixed. This might be due to the RUN_CMD_UTILS +# command - maybe that is returning a zero exit code even though the +# exit code of chgres_cube is nonzero. A similar thing happens in the +# forecast task. +# + ${RUN_CMD_UTILS} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) file for the FV3-LAM for forecast hour fhr failed: diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index afd40d74d2..430aa6a8d4 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -104,7 +104,7 @@ case "$MACHINE" in module list { restore_shell_opts; } > /dev/null 2>&1 NODES=1 - APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" + RUN_CMD_SERIAL="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" ulimit -s unlimited ulimit -a ;; @@ -112,58 +112,24 @@ case "$MACHINE" in "WCOSS_DELL_P3") ulimit -s unlimited ulimit -a - APRUN="mpirun" - ;; - - "HERA") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; - - "ORION") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; - - "JET") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; - - "ODIN") - APRUN="srun -n 1" - ulimit -s unlimited - ulimit -a - ;; - - "CHEYENNE") - APRUN="time" - ;; - - "STAMPEDE") - APRUN="time" - ;; - - "MACOS") - APRUN=time - ;; - - "LINUX") - APRUN=time - ulimit -s unlimited - ulimit -a + RUN_CMD_SERIAL="mpirun" ;; *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +if [ -z ${RUN_CMD_SERIAL:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_SERIAL for your platform" +else + RUN_CMD_SERIAL=$(eval echo ${RUN_CMD_SERIAL}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." +fi # #----------------------------------------------------------------------- # @@ -297,7 +263,7 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" "\ Starting orography file generation..." -$APRUN "${exec_fp}" < "${input_redirect_fn}" || \ +${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: @@ -369,7 +335,7 @@ Please ensure that you've built this executable." print_info_msg "$VERBOSE" " Starting orography file generation..." - $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ + ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the GSL orography GWD data files returned with nonzero exit code: @@ -517,7 +483,7 @@ cd_vrfy "${filter_dir}" print_info_msg "$VERBOSE" " Starting filtering of orography..." -$APRUN "${exec_fp}" || \ +${RUN_CMD_SERIAL} "${exec_fp}" || \ print_err_msg_exit "\ Call to executable that generates filtered orography file returned with non-zero exit code." @@ -586,7 +552,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH0} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$APRUN ${exec_fp} < ${nml_fn} || \ +${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH0}-cell-wide halo from the orography file with a {NHW}-cell-wide halo @@ -612,7 +578,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$APRUN ${exec_fp} < ${nml_fn} || \ +${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH4}-cell-wide halo from the orography file with a {NHW}-cell-wide halo diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index bc51ea137a..ada4929878 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -139,7 +139,7 @@ EOF case "$MACHINE" in "WCOSS_CRAY") - APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} + RUN_CMD_UTILS=${APRUN:-"aprun -j 1 -n 6 -N 6"} ;; "WCOSS_DELL_P3") @@ -150,51 +150,26 @@ case "$MACHINE" in export threads=1 export MP_LABELIO=yes export OMP_NUM_THREADS=$threads - APRUN="mpirun" - ;; - - "HERA") - APRUN="srun" - ;; - - "ORION") - APRUN="srun" - ;; - - "JET") - APRUN="srun" - ;; - - "CHEYENNE") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="mpirun -np $nprocs" - ;; - - "ODIN") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="srun -n $nprocs" - ;; - - "STAMPEDE") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="ibrun -np ${nprocs}" - ;; - - "MACOS") - APRUN=$RUN_CMD_UTILS - ;; - - "LINUX") - APRUN=$RUN_CMD_UTILS + RUN_CMD_UTILS="mpirun" ;; *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + +if [ -z ${RUN_CMD_UTILS:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_UTILS for your platform" +else + RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_UTILS}\'." +fi # #----------------------------------------------------------------------- # @@ -214,7 +189,7 @@ does not exist: Please ensure that you've built this executable." fi -$APRUN ${exec_fp} || \ +${RUN_CMD_UTILS} ${exec_fp} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface climatology files returned with nonzero exit code: diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 513aa66345..10e1d6240c 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -106,71 +106,35 @@ case "$MACHINE" in ulimit -a if [ ${PE_MEMBER01} -gt 24 ];then - APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" + RUN_CMD_FCST="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" else - APRUN="aprun -b -j1 -n${PE_MEMBER01} -N${PE_MEMBER01} -d1 -cc depth" + RUN_CMD_FCST="aprun -b -j1 -n${PE_MEMBER01} -N${PE_MEMBER01} -d1 -cc depth" fi ;; "WCOSS_DELL_P3") ulimit -s unlimited ulimit -a - APRUN="mpirun -l -np ${PE_MEMBER01}" - ;; - - "HERA") - ulimit -s unlimited - ulimit -a - APRUN="srun" - ;; - - "ORION") - ulimit -s unlimited - ulimit -a - APRUN="srun -n ${PE_MEMBER01}" - ;; - - "JET") - ulimit -s unlimited - ulimit -a - APRUN="srun" - ;; - - "ODIN") - module list - ulimit -s unlimited - ulimit -a - APRUN="srun -n ${PE_MEMBER01}" - ;; - - "CHEYENNE") - module list - nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) - APRUN="mpirun -np $nprocs" - ;; - - "STAMPEDE") - module list - APRUN="ibrun -np ${PE_MEMBER01}" - ;; - - "MACOS") - APRUN=$RUN_CMD_FCST - ;; - - "LINUX") - ulimit -s unlimited - ulimit -a - APRUN=$RUN_CMD_FCST + RUN_CMD_FCST="mpirun -l -np ${PE_MEMBER01}" ;; *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) + +if [ -z ${RUN_CMD_FCST:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_FCST for your platform" +else + RUN_CMD_FCST=$(eval echo ${RUN_CMD_FCST}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_FCST}\'." +fi # #----------------------------------------------------------------------- # @@ -555,7 +519,7 @@ Call to function to create a diag table file for the current cycle's # #----------------------------------------------------------------------- # -$APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ +${RUN_CMD_FCST} ${FV3_EXEC_FP} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 025edd46ea..a9b07efef9 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -104,7 +104,7 @@ case "$MACHINE" in export MP_LABELIO=yes export OMP_NUM_THREADS=$threads - APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" + RUN_CMD_POST="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" ;; "WCOSS_DELL_P3") @@ -117,53 +117,25 @@ case "$MACHINE" in export MP_LABELIO=yes export OMP_NUM_THREADS=$threads - APRUN="mpirun" - ;; - - "HERA") - APRUN="srun" - ;; - - "ORION") - ulimit -s unlimited - ulimit -a - APRUN="srun" - ;; - - "JET") - APRUN="srun" - ;; - - "ODIN") - APRUN="srun -n 1" - ;; - - "CHEYENNE") - module list - nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) - APRUN="mpirun -np $nprocs" - ;; - - "STAMPEDE") - nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) - APRUN="ibrun -n $nprocs" - ;; - - "MACOS") - APRUN=$RUN_CMD_POST - ;; - - "LINUX") - APRUN=$RUN_CMD_POST + RUN_CMD_POST="mpirun" ;; *) - print_err_msg_exit "\ -Run command has not been specified for this machine: - MACHINE = \"$MACHINE\"" + source ${MACHINE_FILE} ;; esac + +nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) +if [ -z ${RUN_CMD_POST:-} ] ; then + print_err_msg_exit "\ + Run command was not set in machine file. \ + Please set RUN_CMD_POST for your platform" +else + RUN_CMD_POST=$(eval echo ${RUN_CMD_POST}) + print_info_msg "$VERBOSE" " + All executables will be submitted with command \'${RUN_CMD_POST}\'." +fi # #----------------------------------------------------------------------- # @@ -294,7 +266,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." -${APRUN} ${EXECDIR}/upp.x < itag || print_err_msg_exit "\ +${RUN_CMD_POST} ${EXECDIR}/upp.x < itag || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 15db83c3d3..776c80d280 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -89,6 +89,7 @@ Usage: [use_cron_to_relaunch=\"...\"] \\ [cron_relaunch_intvl_mnts=\"...\"] \\ [verbose=\"...\"] \\ + [machine_file=\"...\"] \\ [stmp=\"...\"] \\ [ptmp=\"...\"] @@ -170,6 +171,10 @@ Note that it is not possible to specify a different value for VERBOSE for each test via this argument; either all tests will have VERBOSE set to \"TRUE\" or all will have it set to \"FALSE\". +machine_file: +Optional argument to set the full path to a machine configuration file. +If not set, a supported platform machine file may be used. + stmp: Argument used to explicitly set the experiment variable STMP in the experiment configuration files of all the WE2E tests the user wants to @@ -225,6 +230,7 @@ valid_args=( \ "use_cron_to_relaunch" \ "cron_relaunch_intvl_mnts" \ "verbose" \ + "machine_file" \ "stmp" \ "ptmp" \ ) @@ -664,6 +670,13 @@ Please correct and rerun." CRON_RELAUNCH_INTVL_MNTS=${cron_relaunch_intvl_mnts:-"02"} VERBOSE=${verbose:-"TRUE"} + MACHINE_FILE=${machine_file:-"${ushdir}/machine/${machine,,}.sh"} + + # Set the machine-specific configuration settings by sourcing the + # machine file in the ush directory + + source $MACHINE_FILE + expt_config_str=${expt_config_str}"\ # # The machine on which to run, the account to which to charge computational @@ -689,6 +702,10 @@ EXPT_SUBDIR=\"${EXPT_SUBDIR}\" USE_CRON_TO_RELAUNCH=\"${USE_CRON_TO_RELAUNCH}\" CRON_RELAUNCH_INTVL_MNTS=\"${CRON_RELAUNCH_INTVL_MNTS}\" # +# Path to machine configuration file. +# +MACHINE_FILE=\"${MACHINE_FILE}\" +# # Flag specifying whether to run in verbose mode. # VERBOSE=\"${VERBOSE}\"" @@ -729,17 +746,9 @@ VERBOSE=\"${VERBOSE}\"" [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ] || \ [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - pregen_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - pregen_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "HERA" ]; then - pregen_basedir="/scratch2/BMC/det/FV3LAM_pregen" - elif [ "$MACHINE" = "JET" ]; then - pregen_basedir="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" - elif [ "$MACHINE" = "CHEYENNE" ]; then - pregen_basedir="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" - else + pregen_basedir=${TEST_PREGEN_BASEDIR:-} + + if [ ! -d "${pregen_basedir:-}" ] ; then print_err_msg_exit "\ The base directory (pregen_basedir) in which the pregenerated grid, orography, and/or surface climatology files are located has not been @@ -837,37 +846,7 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # RUN=\"\${EXPT_SUBDIR}\" envir=\"\${EXPT_SUBDIR}\"" -# -# Set FIXLAM_NCO_BASEDIR. -# - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - FIXLAM_NCO_BASEDIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - FIXLAM_NCO_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" - elif [ "$MACHINE" = "HERA" ]; then - FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" - elif [ "$MACHINE" = "JET" ]; then - FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" - elif [ "$MACHINE" = "CHEYENNE" ]; then - FIXLAM_NCO_BASEDIR="/needs/to/be/specified" - else - print_err_msg_exit "\ -The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, -orography, and surface climatology \"fixed\" files used in NCO mode are -located has not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - expt_config_str=${expt_config_str}" -# -# The base directory in which the pregenerated grid, orography, and surface -# climatology \"fixed\" files used in NCO mode are located. In NCO mode, -# the workflow scripts will create symlinks (in the directory specified -# by FIXLAM) to files in a subdirectory under FIXLAM_NCO_BASDEDIR, where -# the name of the subdirectory is the name of the predefined grid specified -# by PREDEF_GRID_NAME. -# -FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" # # Set COMINgfs if using the FV3GFS or the GSMGFS as the external model # for ICs or LBCs. @@ -877,17 +856,9 @@ FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - COMINgfs="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "HERA" ]; then - COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" - elif [ "$MACHINE" = "JET" ]; then - COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" - elif [ "$MACHINE" = "CHEYENNE" ]; then - COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" - else + COMINgfs=${TEST_COMINgfs:-} + + if [ ! -d "${COMINgfs:-}" ] ; then print_err_msg_exit "\ The directory (COMINgfs) that needs to be specified when running the workflow in NCO mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or @@ -931,19 +902,8 @@ PTMP=\"${PTMP}\"" # if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - extrn_mdl_source_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - extrn_mdl_source_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "HERA" ]; then - extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "JET" ]; then - extrn_mdl_source_basedir="/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "CHEYENNE" ]; then - extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" - elif [ "$MACHINE" = "ORION" ]; then - extrn_mdl_source_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" - else + extrn_mdl_source_basedir=${TEST_EXTRN_MDL_SOURCE_BASEDIR:-} + if [ ! -d "${extrn_mdl_source_basedir:-}" ] ; then print_err_msg_exit "\ The base directory (extrn_mdl_source_basedir) in which the user-staged external model files should be located has not been specified for this @@ -1017,7 +977,7 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" # #----------------------------------------------------------------------- # -# Set MET and MET+ paths, if necessary. +# Check that MET directories have been set appropriately, if needed. # #----------------------------------------------------------------------- # @@ -1026,51 +986,54 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - met_install_dir="/gpfs/hps3/emc/meso/noscrub/emc.metplus/met/10.0.0" - metplus_path="/gpfs/hps3/emc/meso/noscrub/emc.metplus/METplus/METplus-4.0.0" - ccpa_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" - mrms_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" - ndas_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" - met_bin_exec="exec" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - met_install_dir="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" - metplus_path="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" - ccpa_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc" - mrms_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" - ndas_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" - met_bin_exec="exec" - elif [ "$MACHINE" = "HERA" ]; then - met_install_dir="/contrib/met/10.0.0" - metplus_path="/contrib/METplus/METplus-4.0.0" - ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc" - mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc" - ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc" - met_bin_exec="bin" - elif [ "$MACHINE" = "CHEYENNE" ]; then - met_install_dir="/glade/p/ral/jntp/MET/MET_releases/10.0.0" - metplus_path="/glade/p/ral/jntp/MET/METplus/METplus-4.0.0" - ccpa_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc" - mrms_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc" - ndas_obs_dir="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc" - met_bin_exec="bin" - else - print_err_msg_exit "\ -The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) or the observation directories -(CCPA_OBS_DIR, MRMS_OBS_DIR, NDAS_OBS_DIR) have not been specified for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" + check=0 + if [ ! -d ${MET_INSTALL_DIR} ] ; then + print_info_msg "\ + The MET installation location must be set for this machine! + MET_INSTALL_DIR = \"${MET_INSTALL_DIR}\"" + check=1 fi - expt_config_str=${expt_config_str}" -# -# MET and MET+ paths. -# -METPLUS_PATH=\"${metplus_path}\" -MET_INSTALL_DIR=\"${met_install_dir}\" -CCPA_OBS_DIR=\"${ccpa_obs_dir}\" -MRMS_OBS_DIR=\"${mrms_obs_dir}\" -NDAS_OBS_DIR=\"${ndas_obs_dir}\" -MET_BIN_EXEC=\"${met_bin_exec}\"" + if [ ! -d ${METPLUS_PATH} ] ; then + print_info_msg "\ + The MET+ installation location must be set for this machine! + METPLUS_PATH = \"${METPLUS_PATH}\"" + check=1 + fi + + if [ -z ${MET_BIN_EXEC} ] ; then + print_info_msg "\ + The MET execution command must be set for this machine! + MET_BIN_EXEC = \"${MET_BIN_EXEC}\"" + check=1 + fi + + if [ ! -d ${CCPA_OBS_DIR} ] ; then + print_info_msg "\ + The CCPA observation location must be set for this machine! + CCPA_OBS_DIR = \"${CCPA_OBS_DIR}\"" + check=1 + fi + + if [ ! -d ${MRMS_OBS_DIR} ] ; then + print_info_msg "\ + The MRMS observation location must be set for this machine! + MRMS_OBS_DIR = \"${MRMS_OBS_DIR}\"" + check=1 + fi + + if [ ! -d ${NDAS_OBS_DIR} ] ; then + print_info_msg "\ + The NDAS observation location must be set for this machine! + NDAS_OBS_DIR = \"${NDAS_OBS_DIR}\"" + check=1 + fi + + if [ ${check} = 1 ] ; then + print_err_msg_exit "\ + Please set MET variables in the machine file for \ + MACHINE = \"${MACHINE}\"" + fi fi # @@ -1147,16 +1110,11 @@ MAXTRIES_RUN_POST=\"${MAXTRIES_RUN_POST}\"" # #----------------------------------------------------------------------- # - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then + if [ -n "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then if [ "${EXTRN_MDL_SYSBASEDIR_ICS}" = "set_to_non_default_location_in_testing_script" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi + EXTRN_MDL_SYSBASEDIR_ICS="${TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS:-}" if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then print_err_msg_exit "\ @@ -1167,19 +1125,21 @@ initial conditions (EXTRN_MDL_NAME_ICS) combination: EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" fi - else + # Maintain any templates in EXTRN_MDL_SYSBASEDIR_ICS -- don't use + # quotes. + set_bash_param "${expt_config_fp}" \ + "EXTRN_MDL_SYSBASEDIR_ICS" ${EXTRN_MDL_SYSBASEDIR_ICS} - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - print_err_msg_exit "\ + fi + + # Check the base directory for the specified location. + if [ ! -d "$(dirname ${EXTRN_MDL_SYSBASEDIR_ICS%%\$*})" ]; then + print_err_msg_exit "\ The non-default location specified by EXTRN_MDL_SYSBASEDIR_ICS does not exist or is not a directory: EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" - fi - fi - set_bash_param "${expt_config_fp}" \ - "EXTRN_MDL_SYSBASEDIR_ICS" "${EXTRN_MDL_SYSBASEDIR_ICS}" fi # @@ -1189,16 +1149,11 @@ exist or is not a directory: # #----------------------------------------------------------------------- # - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then + if [ -n "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then if [ "${EXTRN_MDL_SYSBASEDIR_LBCS}" = "set_to_non_default_location_in_testing_script" ]; then - EXTRN_MDL_SYSBASEDIR_LBCS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi + EXTRN_MDL_SYSBASEDIR_LBCS="${TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS:-}" if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then print_err_msg_exit "\ @@ -1209,19 +1164,21 @@ initial conditions (EXTRN_MDL_NAME_LBCS) combination: EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" fi - else + # Maintain any templates in EXTRN_MDL_SYSBASEDIR_ICS -- don't use + # quotes. + set_bash_param "${expt_config_fp}" \ + "EXTRN_MDL_SYSBASEDIR_LBCS" ${EXTRN_MDL_SYSBASEDIR_LBCS} - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - print_err_msg_exit "\ + fi + + # Check the base directory for the specified location. + if [ ! -d "$(dirname ${EXTRN_MDL_SYSBASEDIR_LBCS%%\$*})" ]; then + print_err_msg_exit "\ The non-default location specified by EXTRN_MDL_SYSBASEDIR_LBCS does not exist or is not a directory: EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" - fi - fi - set_bash_param "${expt_config_fp}" \ - "EXTRN_MDL_SYSBASEDIR_LBCS" "${EXTRN_MDL_SYSBASEDIR_LBCS}" fi # diff --git a/ush/bash_utils/boolify.sh b/ush/bash_utils/boolify.sh new file mode 100644 index 0000000000..fe32090fda --- /dev/null +++ b/ush/bash_utils/boolify.sh @@ -0,0 +1,69 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function used to change a variety of input boolean +# strings to standard TRUE and FALSE +# +#----------------------------------------------------------------------- +# + +function boolify() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the name of this function and input. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" + local input uc_input + + if [ "$#" -eq 1 ]; then + input="$1" + else + + print_err_msg_exit " +Incorrect number of arguments specified: + + Function name: \"${func_name}\" + Number of arguments specified: $# + +Usage: + + ${func_name} string + +where: + + string: + This is the string that should be converted to TRUE or FALSE +" + fi + + uc_input=$(echo_uppercase $input) + if [ "$uc_input" = "TRUE" ] || \ + [ "$uc_input" = "YES" ]; then + echo "TRUE" + elif [ "$uc_input" = "FALSE" ] || \ + [ "$uc_input" = "NO" ]; then + echo "FALSE" + fi + + # + #----------------------------------------------------------------------- + # + # Restore the shell options saved at the beginning of this script/func- + # tion. + # + #----------------------------------------------------------------------- + # + { restore_shell_opts; } > /dev/null 2>&1 +} diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index afb9428567..e7860e021e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -48,6 +48,11 @@ RUN_ENVIR="nco" # Please see ush/valid_param_vals.sh for a full list of supported # platforms. # +# MACHINE_FILE: +# Path to a configuration file with machine-specific settings. If none +# is provided, setup.sh will attempt to set the path to for a supported +# platform. +# # ACCOUNT: # The account under which to submit jobs to the queue. # @@ -132,6 +137,7 @@ RUN_ENVIR="nco" #----------------------------------------------------------------------- # MACHINE="BIG_COMPUTER" +MACHINE_FILE="" ACCOUNT="project_name" WORKFLOW_MANAGER="none" NCORES_PER_NODE="" @@ -574,12 +580,12 @@ WRITE_DOPOST="FALSE" #----------------------------------------------------------------------- # MODEL="" -MET_INSTALL_DIR="/path/to/MET" +MET_INSTALL_DIR="" MET_BIN_EXEC="bin" -METPLUS_PATH="/path/to/METPlus" -CCPA_OBS_DIR="/path/to/observation-directory/ccpa/proc" -MRMS_OBS_DIR="/path/to/observation-directory/mrms/proc" -NDAS_OBS_DIR="/path/to/observation-directory/ndas/proc" +METPLUS_PATH="" +CCPA_OBS_DIR="" +MRMS_OBS_DIR="" +NDAS_OBS_DIR="" # #----------------------------------------------------------------------- # @@ -663,8 +669,8 @@ FV3GFS_FILE_FMT_LBCS="nemsio" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_ICS="" -EXTRN_MDL_SYSBASEDIR_LBCS="" +EXTRN_MDL_SYSBASEDIR_ICS='' +EXTRN_MDL_SYSBASEDIR_LBCS='' # #----------------------------------------------------------------------- # diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index fdab7dad41..30a6e2d887 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -213,7 +213,7 @@ function get_extrn_mdl_file_dir_info() { hh=${cdate:8:2} mn="00" yyyymmdd=${cdate:0:8} - # Julian day -- not 3 digit day of month + # ddd is the Julian day -- not 3 digit day of month ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) # #----------------------------------------------------------------------- @@ -436,55 +436,11 @@ bination of external model (extrn_mdl_name) and analysis or forecast #----------------------------------------------------------------------- # if [ "${anl_or_fcst}" = "ANL" ]; then - sysbasedir=${EXTRN_MDL_SYSBASEDIR_ICS} + sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_ICS}) elif [ "${anl_or_fcst}" = "FCST" ]; then - sysbasedir=${EXTRN_MDL_SYSBASEDIR_LBCS} + sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_LBCS}) fi - sysdir=$sysbasedir - # Use the basedir unless otherwise specified for special platform - # cases below. - if [ -n "${sysbasedir}" ] ; then - case "${extrn_mdl_name}" in - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "HERA") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "ODIN") - sysdir="$sysbasedir/${yyyymmdd}" - ;; - "CHEYENNE") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" - ;; - esac - ;; - - "RAP") - case "$MACHINE" in - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - esac - ;; - - "HRRR") - case "$MACHINE" in - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - esac - ;; - - esac - fi # #----------------------------------------------------------------------- # diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh new file mode 100755 index 0000000000..d32b1f90da --- /dev/null +++ b/ush/machine/cheyenne.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh}' + ;; + "*") + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Cheyenne. Please set a user-defined file location." + ;; + + esac + echo ${location:-} + +} + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/glade/u/apps/ch/opt/lmod/8.1.7/lmod/8.1.7/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE="${NCORES_PER_NODE:-36}" +SCHED=${SCHED:-"pbspro"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} +QUEUE_HPSS=${QUEUE_HPSS:-"regular"} +QUEUE_FCST=${QUEUE_FCST:-"regular"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} +FIXaer=${FIXaer:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_aer"} +FIXlut=${FIXlut:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS='mpirun -np $nprocs' +RUN_CMD_FCST='mpirun -np $nprocs' +RUN_CMD_POST='mpirun -np $nprocs' + +# MET Installation Locations +MET_INSTALL_DIR="/glade/p/ral/jntp/MET/MET_releases/10.0.0" +METPLUS_PATH="/glade/p/ral/jntp/MET/METplus/METplus-4.0.0" +CCPA_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc" +MRMS_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc" +NDAS_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc" +MET_BIN_EXEC="bin" + +# Test Data Locations +TEST_PREGEN_BASEDIR=/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen +TEST_COMINgfs=/glade/scratch/ketefian/NCO_dirs/COMGFS +TEST_EXTRN_MDL_SOURCE_BASEDIR=/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh new file mode 100755 index 0000000000..864a51c845 --- /dev/null +++ b/ush/machine/hera.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/scratch1/NCEPDEV/rstprod/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' + ;; + *) + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Hera. Will try to pull from HPSS" + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-40} +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} +PARTITION_HPSS=${PARTITION_HPSS:-"service"} +QUEUE_HPSS=${QUEUE_HPSS:-"batch"} +PARTITION_FCST=${PARTITION_FCST:-"hera"} +QUEUE_FCST=${QUEUE_FCST:-"batch"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} +FIXaer=${FIXaer:-"/scratch1/NCEPDEV/global/glopara/fix/fix_aer"} +FIXlut=${FIXlut:-"/scratch1/NCEPDEV/global/glopara/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS="srun" +RUN_CMD_FCST="srun" +RUN_CMD_POST="srun" + +# MET Installation Locations +MET_INSTALL_DIR="/contrib/met/10.0.0" +METPLUS_PATH="/contrib/METplus/METplus-4.0.0" +CCPA_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc" +MRMS_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc" +NDAS_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc" +MET_BIN_EXEC="bin" + +# Test Data Locations +TEST_PREGEN_BASEDIR=/scratch2/BMC/det/FV3LAM_pregen +TEST_COMINgfs=/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS +TEST_EXTRN_MDL_SOURCE_BASEDIR=/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files +TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir +TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir + + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh new file mode 100755 index 0000000000..9b5472ecfd --- /dev/null +++ b/ush/machine/jet.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + "FV3GFS") + case $external_file_fmt in + "nemsio") + location='/public/data/grids/gfs/nemsio' + ;; + "grib2") + location='/public/data/grids/gfs/0p25deg/grib2' + ;; + "netcdf") + location='/public/data/grids/gfs/anl/netcdf/' + ;; + esac + ;; + "RAP") + location='/public/data/grids/rap/full/wrfprs/grib2' + ;; + "HRRR") + location='/public/data/grids/hrrr/conus/wrfprs/grib2' + ;; + "*") + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Jet. Will try to pull from HPSS." + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-24} +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} +PARTITION_HPSS=${PARTITION_HPSS:-"service"} +QUEUE_HPSS=${QUEUE_HPSS:-"batch"} +PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} +QUEUE_FCST=${QUEUE_FCST:-"batch"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} +FIXaer=${FIXaer:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_aer"} +FIXlut=${FIXlut:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS="srun" +RUN_CMD_FCST="srun" +RUN_CMD_POST="srun" + +# Test Data Locations +TEST_PREGEN_BASEDIR=/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen +TEST_COMINgfs=/lfs1/HFIP/hwrf-data/hafs-input/COMGFS +TEST_EXTRN_MDL_SOURCE_BASEDIR=/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files + + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh new file mode 100755 index 0000000000..a9245f5216 --- /dev/null +++ b/ush/machine/odin.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "GSMGFS") + location='/scratch/ywang/EPIC/GDAS/2019053000_mem001' + ;; + "FV3GFS") + location='/scratch/ywang/test_runs/FV3_regional/gfs/${yyyymmdd}' + ;; + "*") + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Odin Please set a user-defined file location." + ;; + esac + echo ${location:-} + +} + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-24} +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} +PARTITION_HPSS=${PARTITION_HPSS:-"workq"} +QUEUE_HPSS=${QUEUE_HPSS:-"workq"} +PARTITION_FCST=${PARTITION_FCST:-"workq"} +QUEUE_FCST=${QUEUE_FCST:-"workq"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} +FIXaer=${FIXaer:-"/scratch/ywang/fix/theia_fix/fix_aer"} +FIXlut=${FIXlut:-"/scratch/ywang/fix/theia_fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} + +RUN_CMD_SERIAL="srun -n 1" +RUN_CMD_UTILS='srun -n $nprocs' +RUN_CMD_FCST='srun -n ${PE_MEMBER01}' +RUN_CMD_POST="srun -n 1" + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh new file mode 100755 index 0000000000..9ed7223752 --- /dev/null +++ b/ush/machine/orion.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + # Orion does not currently have any files staged on disk. + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "*") + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Orion. Please set a user-defined file location." + ;; + + esac + echo ${location:-} + +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-40} +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} +PARTITION_HPSS=${PARTITION_HPSS:-"service"} +QUEUE_HPSS=${QUEUE_HPSS:-"batch"} +PARTITION_FCST=${PARTITION_FCST:-"orion"} +QUEUE_FCST=${QUEUE_FCST:-"batch"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} +FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} +FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS="srun" +RUN_CMD_FCST='srun -n ${PE_MEMBER01}' +RUN_CMD_POST="srun" + +# Test Data Locations +TEST_EXTRN_MDL_SOURCE_BASEDIR=/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh new file mode 100755 index 0000000000..0ba9436805 --- /dev/null +++ b/ush/machine/stampede.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "GSMGFS") + ;& # Fall through. All files in same place + "FV3GFS") + location='/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001' + ;; + "*") + print_err_msg_exit"\ + External model \'${external_model}\' does not have a default + location on Jet Please set a user-defined file location." + ;; + + esac + echo ${location:-} +} + + +SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE="${NCORES_PER_NODE:-68}" +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} +PARTITION_HPSS=${PARTITION_HPSS:-"normal"} +QUEUE_HPSS=${QUEUE_HPSS:-"normal"} +PARTITION_FCST=${PARTITION_FCST:-"normal"} +QUEUE_FCST=${QUEUE_FCST:-"normal"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} +FIXaer=${FIXaer:-"/work/00315/tg455890/stampede2/regional_fv3/fix_aer"} +FIXlut=${FIXlut:-"/work/00315/tg455890/stampede2/regional_fv3/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS='ibrun -np $nprocs' +RUN_CMD_FCST='ibrun -np $nprocs' +RUN_CMD_POST='ibrun -np $nprocs' + +ulimit -s unlimited +ulimit -a diff --git a/ush/machine/wcoss_cray.sh b/ush/machine/wcoss_cray.sh new file mode 100755 index 0000000000..9d2836df5e --- /dev/null +++ b/ush/machine/wcoss_cray.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/gpfs/dell1/nco/ops/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' + ;; + "RAP") + location='/gpfs/hps/nco/ops/com/rap/prod' + ;; + "HRRR") + location='/gpfs/hps/nco/ops/com/hrrr/prod' + ;; + "NAM") + location='/gpfs/dell1/nco/ops/com/nam/prod' + ;; + "*") + print_err_msg_exit"\ + External model \'${external_model}\' does not have a default + location on Jet Please set a user-defined file location." + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/opt/modules/default/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-24} +SCHED=${SCHED:-"lsfcray"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} +QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} +QUEUE_FCST=${QUEUE_FCST:-"dev"} +RELATIVE_LINK_FLAG="" + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} +FIXaer=${FIXaer:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} +FIXlut=${FIXlut:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen"} + +# MET Installation Locations +MET_INSTALL_DIR="/gpfs/hps3/emc/meso/noscrub/emc.metplus/met/10.0.0" +METPLUS_PATH="/gpfs/hps3/emc/meso/noscrub/emc.metplus/METplus/METplus-4.0.0" +CCPA_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" +MRMS_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" +NDAS_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" +MET_BIN_EXEC="exec" + +# Test Data Locations +TEST_PREGEN_BASEDIR=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen +TEST_COMINgfs=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS +TEST_EXTRN_MDL_SOURCE_BASEDIR=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh new file mode 100755 index 0000000000..8ae99fdc1f --- /dev/null +++ b/ush/machine/wcoss_dell_p3.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/gpfs/dell1/nco/ops/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' + ;; + "RAP") + location='/gpfs/hps/nco/ops/com/rap/prod' + ;; + "HRRR") + location='/gpfs/hps/nco/ops/com/hrrr/prod' + ;; + "NAM") + location='/gpfs/dell1/nco/ops/com/nam/prod' + ;; + "*") + print_err_msg_exit"\ + External model \'${external_model}\' does not have a default + location on Jet Please set a user-defined file location." + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System Installations +MODULE_INIT_PATH=${MODULE_INIT_PATH:-/opt/modules/default/init/sh} + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-24} +SCHED=${SCHED:-"lsf"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} +QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} +QUEUE_FCST=${QUEUE_FCST:-"dev"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} +FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} +FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen"} + +# MET Installation Locations +MET_INSTALL_DIR="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" +METPLUS_PATH="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" +CCPA_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc" +MRMS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" +NDAS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" +MET_BIN_EXEC="exec" + + +# Test Data Locations +TEST_PREGEN_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen +TEST_COMINgfs=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS +TEST_EXTRN_MDL_SOURCE_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files + diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 5adf2ac30b..8abba8c334 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -51,6 +51,7 @@ function make_grid_mosaic_file() { "grid_dir" \ "grid_fn" \ "mosaic_fn" \ +"run_cmd" \ ) process_args valid_args "$@" # @@ -166,7 +167,7 @@ Please ensure that you've built this executable." # directory /abc/def and named ghi.nc.nc -- note the double ".nc" # extensions). # - $APRUN "${exec_fp}" \ + ${run_cmd} "${exec_fp}" \ --num_tiles 1 \ --dir "${grid_dir}" \ --tile_file "${grid_fn}" \ diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index e315c240ba..f110a4c26e 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -1,116 +1,10 @@ # #----------------------------------------------------------------------- # -# This file defines and then calls a function that sets known locations -# of files on supported platforms. +# This file sets some parameters that are model or mode specific. # #----------------------------------------------------------------------- # -function set_known_sys_dir() { - - # Usage: - # set_known_sys_dir model - # - # model is the name of the external model - # - local known_sys_dir model_name - - model=$1 - # - #----------------------------------------------------------------------- - # - # Set the system directory (i.e. location on disk, not on HPSS) in - # which the files generated by the external model specified by - # EXTRN_MDL_NAME_ICS that are necessary for generating initial - # condition (IC) and surface files for the FV3SAR are stored (usually - # for a limited time, e.g. for the GFS external model, 2 weeks on - # WCOSS and 2 days on hera). If for a given cycle these files are - # available in this system directory, they will be copied over to a - # subdirectory under the cycle directory. If these files are not - # available in the system directory, then we search for them - # elsewhere, e.g. in the mass store (HPSS). - # - #----------------------------------------------------------------------- - # - - # Set some default known locations on supported platforms. Not all - # platforms have known input locations - case "${model}" in - - "GSMGFS") - case "$MACHINE" in - "ODIN") - known_sys_dir=/scratch/ywang/EPIC/GDAS/2019053000_mem001 - ;; - "CHEYENNE") - known_sys_dir=/glade/p/ral/jntp/UFS_CAM/COMGFS - ;; - "STAMPEDE") - known_sys_dir=/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001 - ;; - esac - ;; - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - ;& # Fall through - "WCOSS_DELL_P3") - known_sys_dir=/gpfs/dell1/nco/ops/com/gfs/prod - ;; - "HERA") - known_sys_dir=/scratch1/NCEPDEV/rstprod/com/gfs/prod - ;; - "JET") - known_sys_dir=/public/data/grids/gfs/nemsio - ;; - "ODIN") - known_sys_dir=/scratch/ywang/test_runs/FV3_regional/gfs - ;; - "STAMPEDE") - known_sys_dir=/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001 - ;; - "CHEYENNE") - known_sys_dir=/glade/p/ral/jntp/UFS_CAM/COMGFS} - ;; - esac - ;; - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - ;& # Fall through - "WCOSS_DELL_P3") - known_sys_dir=/gpfs/hps/nco/ops/com/rap/prod - ;; - esac - ;; - - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - ;& # Fall through - "WCOSS_DELL_P3") - known_sys_dir=/gpfs/hps/nco/ops/com/hrrr/prod - ;; - esac - ;; - - "NAM") - case "$MACHINE" in - "WCOSS_CRAY") - ;& # Fall through - "WCOSS_DELL_P3") - known_sys_dir=/gpfs/dell1/nco/ops/com/nam/prod - ;; - esac - ;; - - esac - - echo $known_sys_dir -} - function set_extrn_mdl_params() { # #----------------------------------------------------------------------- @@ -147,6 +41,7 @@ function set_extrn_mdl_params() { ;; esac } + # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index d8db064c69..74426ef7d8 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -50,6 +50,7 @@ cd_vrfy ${scrfunc_dir} # #----------------------------------------------------------------------- # + . ./source_util_funcs.sh print_info_msg " @@ -132,330 +133,77 @@ fi # #----------------------------------------------------------------------- # -# Make sure that RUN_ENVIR is set to a valid value. +# Make sure that user-defined variables are set to valid values # -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_ENVIR" "valid_vals_RUN_ENVIR" -# -#----------------------------------------------------------------------- -# -# Make sure that VERBOSE is set to a valid value. +# Set binary switch variables to either "TRUE" or "FALSE" by calling +# boolify so we don't have to consider other valid values later on # #----------------------------------------------------------------------- # +check_var_valid_value "RUN_ENVIR" "valid_vals_RUN_ENVIR" + check_var_valid_value "VERBOSE" "valid_vals_VERBOSE" -# -# Set VERBOSE to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -VERBOSE=$(echo_uppercase $VERBOSE) -if [ "$VERBOSE" = "TRUE" ] || \ - [ "$VERBOSE" = "YES" ]; then - VERBOSE="TRUE" -elif [ "$VERBOSE" = "FALSE" ] || \ - [ "$VERBOSE" = "NO" ]; then - VERBOSE="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DEBUG is set to a valid value. -# -#----------------------------------------------------------------------- -# +VERBOSE=$(boolify $VERBOSE) + check_var_valid_value "DEBUG" "valid_vals_DEBUG" -# -# Set DEBUG to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -DEBUG=$(echo_uppercase $DEBUG) -if [ "$DEBUG" = "TRUE" ] || \ - [ "$DEBUG" = "YES" ]; then - DEBUG="TRUE" -elif [ "$DEBUG" = "FALSE" ] || \ - [ "$DEBUG" = "NO" ]; then - DEBUG="FALSE" -fi +DEBUG=$(boolify $DEBUG) # #----------------------------------------------------------------------- # -# If DEBUG is set to "TRUE" but VERBOSE is set to "FALSE", reset VERBOSE -# to "TRUE" to print out all of the VERBOSE output (in addition to any -# DEBUG output). +# If DEBUG is set to "TRUE", set VERBOSE to "TRUE" to print out all +# of the VERBOSE output (in addition to any DEBUG output). # #----------------------------------------------------------------------- # -if [ "$DEBUG" = "TRUE" ] && [ "$VERBOSE" = "FALSE" ]; then +if [ "$DEBUG" = "TRUE" ]; then print_info_msg " -Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." - VERBOSE="TRUE" +Setting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." + VERBOSE="TRUE" fi -# -#----------------------------------------------------------------------- -# -# Make sure that USE_CRON_TO_RELAUNCH is set to a valid value. -# -#----------------------------------------------------------------------- -# + check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_USE_CRON_TO_RELAUNCH" -# -# Set USE_CRON_TO_RELAUNCH to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -USE_CRON_TO_RELAUNCH=$(echo_uppercase $USE_CRON_TO_RELAUNCH) -if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ] || \ - [ "${USE_CRON_TO_RELAUNCH}" = "YES" ]; then - USE_CRON_TO_RELAUNCH="TRUE" -elif [ "${USE_CRON_TO_RELAUNCH}" = "FALSE" ] || \ - [ "${USE_CRON_TO_RELAUNCH}" = "NO" ]; then - USE_CRON_TO_RELAUNCH="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_GRID is set to a valid value. -# -#----------------------------------------------------------------------- -# +USE_CRON_TO_RELAUNCH=$(boolify $USE_CRON_TO_RELAUNCH) + check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_RUN_TASK_MAKE_GRID" -# -# Set RUN_TASK_MAKE_GRID to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_MAKE_GRID=$(echo_uppercase $RUN_TASK_MAKE_GRID) -if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_GRID}" = "YES" ]; then - RUN_TASK_MAKE_GRID="TRUE" -elif [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_GRID}" = "NO" ]; then - RUN_TASK_MAKE_GRID="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_OROG is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_MAKE_GRID=$(boolify $RUN_TASK_MAKE_GRID) + check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_RUN_TASK_MAKE_OROG" -# -# Set RUN_TASK_MAKE_OROG to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_MAKE_OROG=$(echo_uppercase $RUN_TASK_MAKE_OROG) -if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_OROG}" = "YES" ]; then - RUN_TASK_MAKE_OROG="TRUE" -elif [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_OROG}" = "NO" ]; then - RUN_TASK_MAKE_OROG="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_SFC_CLIMO is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_MAKE_OROG=$(boolify $RUN_TASK_MAKE_OROG) + check_var_valid_value \ "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_RUN_TASK_MAKE_SFC_CLIMO" -# -# Set RUN_TASK_MAKE_SFC_CLIMO to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -RUN_TASK_MAKE_SFC_CLIMO=$(echo_uppercase $RUN_TASK_MAKE_SFC_CLIMO) -if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "YES" ]; then - RUN_TASK_MAKE_SFC_CLIMO="TRUE" -elif [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "NO" ]; then - RUN_TASK_MAKE_SFC_CLIMO="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_RUN_POST is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_MAKE_SFC_CLIMO=$(boolify $RUN_TASK_MAKE_SFC_CLIMO) + check_var_valid_value \ "RUN_TASK_RUN_POST" "valid_vals_RUN_TASK_RUN_POST" -# -# Set RUN_TASK_RUN_POST to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -RUN_TASK_RUN_POST=$(echo_uppercase $RUN_TASK_RUN_POST) -if [ "${RUN_TASK_RUN_POST}" = "TRUE" ] || \ - [ "${RUN_TASK_RUN_POST}" = "YES" ]; then - RUN_TASK_RUN_POST="TRUE" -elif [ "${RUN_TASK_RUN_POST}" = "FALSE" ] || \ - [ "${RUN_TASK_RUN_POST}" = "NO" ]; then - RUN_TASK_RUN_POST="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_GRIDSTAT is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_RUN_POST=$(boolify $RUN_TASK_RUN_POST) + check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_RUN_TASK_VX_GRIDSTAT" -# -# Set RUN_TASK_VX_GRIDSTAT to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_GRIDSTAT=$(echo_uppercase $RUN_TASK_VX_GRIDSTAT) -if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "YES" ]; then - RUN_TASK_VX_GRIDSTAT="TRUE" -elif [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "NO" ]; then - RUN_TASK_VX_GRIDSTAT="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_POINTSTAT is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_VX_GRIDSTAT=$(boolify $RUN_TASK_VX_GRIDSTAT) + check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_RUN_TASK_VX_POINTSTAT" -# -# Set RUN_TASK_VX_POINTSTAT to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_POINTSTAT=$(echo_uppercase $RUN_TASK_VX_POINTSTAT) -if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "YES" ]; then - RUN_TASK_VX_POINTSTAT="TRUE" -elif [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "NO" ]; then - RUN_TASK_VX_POINTSTAT="FALSE" -fi +RUN_TASK_VX_POINTSTAT=$(boolify $RUN_TASK_VX_POINTSTAT) -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_ENSGRID is set to a valid value. -# -#----------------------------------------------------------------------- -# check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_RUN_TASK_VX_ENSGRID" -# -# Set RUN_TASK_VX_ENSGRID to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_ENSGRID=$(echo_uppercase $RUN_TASK_VX_ENSGRID) -if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSGRID}" = "YES" ]; then - RUN_TASK_VX_ENSGRID="TRUE" -elif [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_ENSGRID}" = "NO" ]; then - RUN_TASK_VX_ENSGRID="FALSE" -fi +RUN_TASK_VX_ENSGRID=$(boolify $RUN_TASK_VX_ENSGRID) -# -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_ENSPOINT is set to a valid value. -# -#----------------------------------------------------------------------- -# check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_RUN_TASK_VX_ENSPOINT" -# -# Set RUN_TASK_VX_ENSPOINT to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_ENSPOINT=$(echo_uppercase $RUN_TASK_VX_ENSPOINT) -if [ "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSPOINT}" = "YES" ]; then - RUN_TASK_VX_ENSPOINT="TRUE" -elif [ "${RUN_TASK_VX_ENSPOINT}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_ENSPOINT}" = "NO" ]; then - RUN_TASK_VX_ENSPOINT="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SHUM is set to a valid value. -# -#----------------------------------------------------------------------- -# +RUN_TASK_VX_ENSPOINT=$(boolify $RUN_TASK_VX_ENSPOINT) + +check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" +USE_FVCOM=$(boolify $USE_FVCOM) + check_var_valid_value "DO_SHUM" "valid_vals_DO_SHUM" -# -# Set DO_SHUM to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SHUM=$(echo_uppercase $DO_SHUM) -if [ "${DO_SHUM}" = "TRUE" ] || \ - [ "${DO_SHUM}" = "YES" ]; then - DO_SHUM="TRUE" -elif [ "${DO_SHUM}" = "FALSE" ] || \ - [ "${DO_SHUM}" = "NO" ]; then - DO_SHUM="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SPPT is set to a valid value. -# -#----------------------------------------------------------------------- -# +DO_SHUM=$(boolify $DO_SHUM) + check_var_valid_value "DO_SPPT" "valid_vals_DO_SPPT" -# -# Set DO_SPPT to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SPPT=$(echo_uppercase $DO_SPPT) -if [ "${DO_SPPT}" = "TRUE" ] || \ - [ "${DO_SPPT}" = "YES" ]; then - DO_SPPT="TRUE" -elif [ "${DO_SPPT}" = "FALSE" ] || \ - [ "${DO_SPPT}" = "NO" ]; then - DO_SPPT="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SKEB is set to a valid value. -# -#----------------------------------------------------------------------- -# +DO_SPPT=$(boolify $DO_SPPT) + check_var_valid_value "DO_SKEB" "valid_vals_DO_SKEB" -# -# Set DO_SKEB to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SKEB=$(echo_uppercase $DO_SKEB) -if [ "${DO_SKEB}" = "TRUE" ] || \ - [ "${DO_SKEB}" = "YES" ]; then - DO_SKEB="TRUE" -elif [ "${DO_SKEB}" = "FALSE" ] || \ - [ "${DO_SKEB}" = "NO" ]; then - DO_SKEB="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SPP is set to a valid value. -# -#----------------------------------------------------------------------- -# +DO_SKEB=$(boolify $DO_SKEB) + check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" -# -# Set DO_SPP to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SPP=$(echo_uppercase $DO_SPP) -if [ "${DO_SPP}" = "TRUE" ] || \ - [ "${DO_SPP}" = "YES" ]; then - DO_SPP="TRUE" -elif [ "${DO_SPP}" = "FALSE" ] || \ - [ "${DO_SPP}" = "NO" ]; then - DO_SPP="FALSE" -fi +DO_SPP=$(boolify $DO_SPP) # #----------------------------------------------------------------------- # @@ -491,6 +239,19 @@ fi # #----------------------------------------------------------------------- # +check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" +SUB_HOURLY_POST=$(boolify $SUB_HOURLY_POST) +# +#----------------------------------------------------------------------- +# +# Make sure that DOT_OR_USCORE is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DOT_OR_USCORE" "valid_vals_DOT_OR_USCORE" +# +#----------------------------------------------------------------------- +# # Make sure that USE_FVCOM is set to a valid value and assign directory # and file names. # @@ -499,49 +260,153 @@ fi #----------------------------------------------------------------------- # check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" +USE_FVCOM=$(boolify $USE_FVCOM) check_var_valid_value "FVCOM_WCSTART" "valid_vals_FVCOM_WCSTART" +FVCOM_WCSTART=$(echo_lowercase $FVCOM_WCSTART) # -# Set USE_FVCOM to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. +#----------------------------------------------------------------------- # -USE_FVCOM=$(echo_uppercase $USE_FVCOM) -if [ "$USE_FVCOM" = "TRUE" ] || \ - [ "$USE_FVCOM" = "YES" ]; then - USE_FVCOM="TRUE" -elif [ "$USE_FVCOM" = "FALSE" ] || \ - [ "$USE_FVCOM" = "NO" ]; then - USE_FVCOM="FALSE" -fi -FVCOM_WCSTART=$(echo_lowercase $FVCOM_WCSTART) +# Set various directories. +# +# HOMErrfs: +# Top directory of the clone of the FV3-LAM workflow git repository. +# +# USHDIR: +# Directory containing the shell scripts called by the workflow. +# +# SCRIPTSDIR: +# Directory containing the ex scripts called by the workflow. +# +# JOBSSDIR: +# Directory containing the jjobs scripts called by the workflow. +# +# SORCDIR: +# Directory containing various source codes. +# +# PARMDIR: +# Directory containing parameter files, template files, etc. +# +# EXECDIR: +# Directory containing various executable files. +# +# TEMPLATE_DIR: +# Directory in which templates of various FV3-LAM input files are locat- +# ed. +# +# UFS_WTHR_MDL_DIR: +# Directory in which the (NEMS-enabled) FV3-LAM application is located. +# This directory includes subdirectories for FV3, NEMS, and FMS. # #----------------------------------------------------------------------- # -# Make sure that SUB_HOURLY_POST and DT_SUBHOURLY_POST_MNTS are set to -# valid values. + +# +# The current script should be located in the ush subdirectory of the +# workflow directory. Thus, the workflow directory is the one above the +# directory of the current script. +# +SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} + # #----------------------------------------------------------------------- # -check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" +# Set the base directories in which codes obtained from external reposi- +# tories (using the manage_externals tool) are placed. Obtain the rela- +# tive paths to these directories by reading them in from the manage_ex- +# ternals configuration file. (Note that these are relative to the lo- +# cation of the configuration file.) Then form the full paths to these +# directories. Finally, make sure that each of these directories actu- +# ally exists. # -# Set SUB_HOURLY_POST to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. +#----------------------------------------------------------------------- +# +mng_extrns_cfg_fn=$( $READLINK -f "${SR_WX_APP_TOP_DIR}/Externals.cfg" ) +property_name="local_path" +# +# Get the path to the workflow scripts # -SUB_HOURLY_POST=$(echo_uppercase $SUB_HOURLY_POST) -if [ "${SUB_HOURLY_POST}" = "TRUE" ] || \ - [ "${SUB_HOURLY_POST}" = "YES" ]; then - SUB_HOURLY_POST="TRUE" -elif [ "${SUB_HOURLY_POST}" = "FALSE" ] || \ - [ "${SUB_HOURLY_POST}" = "NO" ]; then - SUB_HOURLY_POST="FALSE" +external_name=regional_workflow +HOMErrfs=$( \ +get_manage_externals_config_property \ +"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ +print_err_msg_exit "\ +Call to function get_manage_externals_config_property failed." +HOMErrfs="${SR_WX_APP_TOP_DIR}/${HOMErrfs}" +set +x +# +# Get the base directory of the FV3 forecast model code. +# +external_name="${FCST_MODEL}" +UFS_WTHR_MDL_DIR=$( \ +get_manage_externals_config_property \ +"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ +print_err_msg_exit "\ +Call to function get_manage_externals_config_property failed." + +UFS_WTHR_MDL_DIR="${SR_WX_APP_TOP_DIR}/${UFS_WTHR_MDL_DIR}" +if [ ! -d "${UFS_WTHR_MDL_DIR}" ]; then + print_err_msg_exit "\ +The base directory in which the FV3 source code should be located +(UFS_WTHR_MDL_DIR) does not exist: + UFS_WTHR_MDL_DIR = \"${UFS_WTHR_MDL_DIR}\" +Please clone the external repository containing the code in this directory, +build the executable, and then rerun the workflow." +fi +# +# Get the base directory of the UFS_UTILS codes. +# +external_name="ufs_utils" +UFS_UTILS_DIR=$( \ +get_manage_externals_config_property \ +"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ +print_err_msg_exit "\ +Call to function get_manage_externals_config_property failed." + +UFS_UTILS_DIR="${SR_WX_APP_TOP_DIR}/${UFS_UTILS_DIR}" +if [ ! -d "${UFS_UTILS_DIR}" ]; then + print_err_msg_exit "\ +The base directory in which the UFS utilities source codes should be lo- +cated (UFS_UTILS_DIR) does not exist: + UFS_UTILS_DIR = \"${UFS_UTILS_DIR}\" +Please clone the external repository containing the code in this direct- +ory, build the executables, and then rerun the workflow." fi # -#----------------------------------------------------------------------- +# Get the base directory of the UPP code. # -# Make sure that DOT_OR_USCORE is set to a valid value. +external_name="UPP" +UPP_DIR=$( \ +get_manage_externals_config_property \ +"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ +print_err_msg_exit "\ +Call to function get_manage_externals_config_property failed." + +UPP_DIR="${SR_WX_APP_TOP_DIR}/${UPP_DIR}" +if [ ! -d "${UPP_DIR}" ]; then + print_err_msg_exit "\ +The base directory in which the UPP source code should be located +(UPP_DIR) does not exist: + UPP_DIR = \"${UPP_DIR}\" +Please clone the external repository containing the code in this directory, +build the executable, and then rerun the workflow." +fi + # -#----------------------------------------------------------------------- +# Define some other useful paths # -check_var_valid_value "DOT_OR_USCORE" "valid_vals_DOT_OR_USCORE" +USHDIR="$HOMErrfs/ush" +SCRIPTSDIR="$HOMErrfs/scripts" +JOBSDIR="$HOMErrfs/jobs" +SORCDIR="$HOMErrfs/sorc" +SRC_DIR="${SR_WX_APP_TOP_DIR}/src" +PARMDIR="$HOMErrfs/parm" +MODULES_DIR="$HOMErrfs/modulefiles" +EXECDIR="${SR_WX_APP_TOP_DIR}/bin" +TEMPLATE_DIR="$USHDIR/templates" +VX_CONFIG_DIR="$TEMPLATE_DIR/parm" +METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" +MET_CONFIG="$TEMPLATE_DIR/parm/met" + # #----------------------------------------------------------------------- # @@ -555,149 +420,35 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" # #----------------------------------------------------------------------- # -# Set the number of cores per node, the job scheduler, and the names of -# several queues. These queues are defined in the default and local -# workflow/experiment configuration script. -# -# Also, set the machine-dependent flag RELATIVE_OR_NULL that specifies -# the flag to pass to the link creation command (ln_vrfy) when attempting -# to create relative symlinks. On machines that don't support relative -# symlinks, it should be set to a null string. -# -#----------------------------------------------------------------------- -# -RELATIVE_LINK_FLAG="" -case $MACHINE in - - "WCOSS_CRAY") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-24}" - SCHED="lsfcray" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} - QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_FCST=${QUEUE_FCST:-"dev"} -# - RELATIVE_LINK_FLAG="" - ;; - - "WCOSS_DELL_P3") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-24}" - SCHED="lsf" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} - QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_FCST=${QUEUE_FCST:-"dev"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "HERA") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-40}" - SCHED=${SCHED:-"slurm"} - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} - QUEUE_HPSS=${QUEUE_HPSS:-"batch"} - PARTITION_FCST=${PARTITION_FCST:-"hera"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "ORION") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-40}" - SCHED=${SCHED:-"slurm"} - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} - QUEUE_HPSS=${QUEUE_HPSS:-"batch"} - PARTITION_FCST=${PARTITION_FCST:-"orion"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "JET") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-24}" - SCHED=${SCHED:-"slurm"} - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} - QUEUE_HPSS=${QUEUE_HPSS:-"batch"} - PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "ODIN") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-24}" - SCHED=${SCHED:-"slurm"} - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} - PARTITION_HPSS=${PARTITION_HPSS:-"workq"} - QUEUE_HPSS=${QUEUE_HPSS:-"workq"} - PARTITION_FCST=${PARTITION_FCST:-"workq"} - QUEUE_FCST=${QUEUE_FCST:-"workq"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "CHEYENNE") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-36}" - SCHED=${SCHED:-"pbspro"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} - QUEUE_HPSS=${QUEUE_HPSS:-"regular"} - QUEUE_FCST=${QUEUE_FCST:-"regular"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "STAMPEDE") - WORKFLOW_MANAGER="rocoto" - NCORES_PER_NODE="${NCORES_PER_NODE:-68}" - SCHED="slurm" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} - PARTITION_HPSS=${PARTITION_HPSS:-"normal"} - QUEUE_HPSS=${QUEUE_HPSS:-"normal"} - PARTITION_FCST=${PARTITION_FCST:-"normal"} - QUEUE_FCST=${QUEUE_FCST:-"normal"} -# - RELATIVE_LINK_FLAG="--relative" - ;; - - "MACOS") - WORKFLOW_MANAGER="none" - SCHED="none" - ;; - - "LINUX") - WORKFLOW_MANAGER=${WORKFLOW_MANAGER:-"none"} - SCHED=${SCHED:-"none"} - ;; - - "*") - NCORES_PER_NODE="2" # Need some arbitrary default value to avoid division by zero errors - - print_err_msg_exit "\ - You are running on an unknown platform! MACHINE=${MACHINE} is not a valid - choice." - ;; +# Source the machine config file containing architechture information, +# queue names, and supported input file paths. +# +#----------------------------------------------------------------------- +# +RELATIVE_LINK_FLAG="--relative" +MACHINE_FILE=${MACHINE_FILE:-${USHDIR}/machine/$(echo_lowercase $MACHINE).sh} +source ${MACHINE_FILE} -esac +if [ -z "${NCORES_PER_NODE:-}" ]; then + print_err_msg_exit "\ + NCORES_PER_NODE has not been specified in the file ${MACHINE_FILE} + Please ensure this value has been set for your desired platform. " +fi -if [ -z "$NCORES_PER_NODE" ]; then - print_err_msg_exit "\ - NCORES_PER_NODE is a required setting for your platform! Please - set it in config.sh. - MACHINE = ${MACHINE}" +if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then + print_err_msg_exit "\ +One or more fix file directories have not been specified for this machine: + MACHINE = \"$MACHINE\" + FIXgsm = \"${FIXgsm:-\"\"} + FIXaer = \"${FIXaer:-\"\"} + FIXlut = \"${FIXlut:-\"\"} + TOPO_DIR = \"${TOPO_DIR:-\"\"} + SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} + FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} +You can specify the missing location(s) in ${machine_file}" fi + + # #----------------------------------------------------------------------- # @@ -817,14 +568,7 @@ check_var_valid_value "USE_MERRA_CLIMO" "valid_vals_USE_MERRA_CLIMO" # Set USE_MERRA_CLIMO to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -USE_MERRA_CLIMO=$(echo_uppercase $USE_MERRA_CLIMO) -if [ "${USE_MERRA_CLIMO}" = "TRUE" ] || \ - [ "${USE_MERRA_CLIMO}" = "YES" ]; then - USE_MERRA_CLIMO="TRUE" -elif [ "${USE_MERRA_CLIMO}" = "FALSE" ] || \ - [ "${USE_MERRA_CLIMO}" = "NO" ]; then - USE_MERRA_CLIMO="FALSE" -fi +USE_MERRA_CLIMO=$(boolify $USE_MERRA_CLIMO) # Force to "TRUE" in case of FV3_GFS_v15_thompson_mynn_lam3km: if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15_thompson_mynn_lam3km" ]; then USE_MERRA_CLIMO="TRUE" @@ -991,248 +735,7 @@ if [ $NUM_CYCLES -gt 90 ] ; then Too many cycles in ALL_CDATES to list, redefining in abbreviated form." ALL_CDATES="${DATE_FIRST_CYCL}${CYCL_HRS[0]}...${DATE_LAST_CYCL}${CYCL_HRS[-1]}" fi -# -#----------------------------------------------------------------------- -# -# Set various directories. -# -# HOMErrfs: -# Top directory of the clone of the FV3-LAM workflow git repository. -# -# USHDIR: -# Directory containing the shell scripts called by the workflow. -# -# SCRIPTSDIR: -# Directory containing the ex scripts called by the workflow. -# -# JOBSSDIR: -# Directory containing the jjobs scripts called by the workflow. -# -# SORCDIR: -# Directory containing various source codes. -# -# PARMDIR: -# Directory containing parameter files, template files, etc. -# -# EXECDIR: -# Directory containing various executable files. -# -# TEMPLATE_DIR: -# Directory in which templates of various FV3-LAM input files are located. -# -# UFS_WTHR_MDL_DIR: -# Directory in which the (NEMS-enabled) FV3-LAM application is located. -# This directory includes subdirectories for FV3, NEMS, and FMS. -# -#----------------------------------------------------------------------- -# - -# -# The current script should be located in the ush subdirectory of the -# workflow directory. Thus, the workflow directory is the one above the -# directory of the current script. Get the path to this latter directo- -# ry and save it in HOMErrfs. -# -SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} -# GSK to do: Get HOMErrfs from Externals.cfg -HOMErrfs="${SR_WX_APP_TOP_DIR}/regional_workflow" - -USHDIR="$HOMErrfs/ush" -SCRIPTSDIR="$HOMErrfs/scripts" -JOBSDIR="$HOMErrfs/jobs" -SORCDIR="$HOMErrfs/sorc" -SRC_DIR="${SR_WX_APP_TOP_DIR}/src" -PARMDIR="$HOMErrfs/parm" -MODULES_DIR="$HOMErrfs/modulefiles" -EXECDIR="${SR_WX_APP_TOP_DIR}/bin" -TEMPLATE_DIR="$USHDIR/templates" -VX_CONFIG_DIR="$TEMPLATE_DIR/parm" -METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" -MET_CONFIG="$TEMPLATE_DIR/parm/met" - -case "$MACHINE" in - - "WCOSS_CRAY") - FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - FIXaer=${FIXaer:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} - FIXlut=${FIXlut:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "WCOSS_DELL_P3") - FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} - FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "HERA") - FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} - FIXaer=${FIXaer:-"/scratch1/NCEPDEV/global/glopara/fix/fix_aer"} - FIXlut=${FIXlut:-"/scratch1/NCEPDEV/global/glopara/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} - ;; - - "ORION") - FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} - FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} - FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "JET") - FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} - FIXaer=${FIXaer:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_aer"} - FIXlut=${FIXlut:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "ODIN") - FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} - FIXaer=${FIXaer:-"/scratch/ywang/fix/theia_fix/fix_aer"} - FIXlut=${FIXlut:-"/scratch/ywang/fix/theia_fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "CHEYENNE") - FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} - FIXaer=${FIXaer:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_aer"} - FIXlut=${FIXlut:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - "STAMPEDE") - FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} - FIXaer=${FIXaer:-"/work/00315/tg455890/stampede2/regional_fv3/fix_aer"} - FIXlut=${FIXlut:-"/work/00315/tg455890/stampede2/regional_fv3/fix_lut"} - TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} - FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} - ;; - - *) - if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then - print_err_msg_exit "\ -One or more fix file directories have not been specified for this machine: - MACHINE = \"$MACHINE\" - FIXgsm = \"${FIXgsm:-\"\"} - FIXaer = \"${FIXaer:-\"\"} - FIXlut = \"${FIXlut:-\"\"} - TOPO_DIR = \"${TOPO_DIR:-\"\"} - SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} -You can specify the missing location(s) in ${EXPT_CONFIG_FN}." - fi - ;; - -esac -# -#----------------------------------------------------------------------- -# -# Set the base directories in which codes obtained from external reposi- -# tories (using the manage_externals tool) are placed. Obtain the rela- -# tive paths to these directories by reading them in from the manage_ex- -# ternals configuration file. (Note that these are relative to the lo- -# cation of the configuration file.) Then form the full paths to these -# directories. Finally, make sure that each of these directories actu- -# ally exists. -# -#----------------------------------------------------------------------- -# -mng_extrns_cfg_fn=$( $READLINK -f "${SR_WX_APP_TOP_DIR}/Externals.cfg" ) -property_name="local_path" -# -# Get the base directory of the FV3 forecast model code. -# -external_name="${FCST_MODEL}" -UFS_WTHR_MDL_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." - -UFS_WTHR_MDL_DIR="${SR_WX_APP_TOP_DIR}/${UFS_WTHR_MDL_DIR}" -if [ ! -d "${UFS_WTHR_MDL_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the FV3 source code should be located -(UFS_WTHR_MDL_DIR) does not exist: - UFS_WTHR_MDL_DIR = \"${UFS_WTHR_MDL_DIR}\" -Please clone the external repository containing the code in this directory, -build the executable, and then rerun the workflow." -fi -# -# Get the base directory of the UFS_UTILS codes. -# -external_name="ufs_utils" -UFS_UTILS_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." - -UFS_UTILS_DIR="${SR_WX_APP_TOP_DIR}/${UFS_UTILS_DIR}" -if [ ! -d "${UFS_UTILS_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the UFS utilities source codes should be lo- -cated (UFS_UTILS_DIR) does not exist: - UFS_UTILS_DIR = \"${UFS_UTILS_DIR}\" -Please clone the external repository containing the code in this direct- -ory, build the executables, and then rerun the workflow." -fi -# -# Get the base directory of the UPP code. -# -external_name="UPP" -UPP_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." -UPP_DIR="${SR_WX_APP_TOP_DIR}/${UPP_DIR}" -if [ ! -d "${UPP_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the UPP source code should be located -(UPP_DIR) does not exist: - UPP_DIR = \"${UPP_DIR}\" -Please clone the external repository containing the code in this directory, -build the executable, and then rerun the workflow." -fi -# -#----------------------------------------------------------------------- -# -# Make sure that USE_CUSTOM_POST_CONFIG_FILE is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value \ - "USE_CUSTOM_POST_CONFIG_FILE" "valid_vals_USE_CUSTOM_POST_CONFIG_FILE" -# -# Set USE_CUSTOM_POST_CONFIG_FILE to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase ${USE_CUSTOM_POST_CONFIG_FILE}) -if [ "$USE_CUSTOM_POST_CONFIG_FILE" = "TRUE" ] || \ - [ "$USE_CUSTOM_POST_CONFIG_FILE" = "YES" ]; then - USE_CUSTOM_POST_CONFIG_FILE="TRUE" -elif [ "$USE_CUSTOM_POST_CONFIG_FILE" = "FALSE" ] || \ - [ "$USE_CUSTOM_POST_CONFIG_FILE" = "NO" ]; then - USE_CUSTOM_POST_CONFIG_FILE="FALSE" -fi # #----------------------------------------------------------------------- # @@ -1739,24 +1242,11 @@ FIELD_TABLE_FP="${EXPTDIR}/${FIELD_TABLE_FN}" FV3_NML_FN="${FV3_NML_BASE_SUITE_FN%.*}" FV3_NML_FP="${EXPTDIR}/${FV3_NML_FN}" NEMS_CONFIG_FP="${EXPTDIR}/${NEMS_CONFIG_FN}" -# -#----------------------------------------------------------------------- -# -# Make sure that USE_USER_STAGED_EXTRN_FILES is set to a valid value. -# -#----------------------------------------------------------------------- -# + + check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_USE_USER_STAGED_EXTRN_FILES" -# -# Set USE_USER_STAGED_EXTRN_FILES to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -USE_USER_STAGED_EXTRN_FILES=$(echo_uppercase $USE_USER_STAGED_EXTRN_FILES) -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "YES" ]; then - USE_USER_STAGED_EXTRN_FILES="TRUE" -elif [ "${USE_USER_STAGED_EXTRN_FILES}" = "NO" ]; then - USE_USER_STAGED_EXTRN_FILES="FALSE" -fi +USE_USER_STAGED_EXTRN_FILES=$(boolify $USE_USER_STAGED_EXTRN_FILES) + # #----------------------------------------------------------------------- # @@ -1794,18 +1284,7 @@ fi #----------------------------------------------------------------------- # check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" -# -# Set DO_ENSEMBLE to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -DO_ENSEMBLE=$(echo_uppercase $DO_ENSEMBLE) -if [ "$DO_ENSEMBLE" = "TRUE" ] || \ - [ "$DO_ENSEMBLE" = "YES" ]; then - DO_ENSEMBLE="TRUE" -elif [ "$DO_ENSEMBLE" = "FALSE" ] || \ - [ "$DO_ENSEMBLE" = "NO" ]; then - DO_ENSEMBLE="FALSE" -fi +DO_ENSEMBLE=$(boolify $DO_ENSEMBLE) NDIGITS_ENSMEM_NAMES="0" ENSMEM_NAMES=("") @@ -2431,10 +1910,9 @@ check_var_valid_value "WRITE_DOPOST" "valid_vals_WRITE_DOPOST" # Set WRITE_DOPOST to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. # -WRITE_DOPOST=$(echo_uppercase $WRITE_DOPOST) -if [ "$WRITE_DOPOST" = "TRUE" ] || \ - [ "$WRITE_DOPOST" = "YES" ]; then - WRITE_DOPOST="TRUE" +WRITE_DOPOST=$(boolify $WRITE_DOPOST) + +if [ "$WRITE_DOPOST" = "TRUE" ] ; then # Turn off run_post RUN_TASK_RUN_POST="FALSE" @@ -2444,51 +1922,15 @@ if [ "$WRITE_DOPOST" = "TRUE" ] || \ print_err_msg_exit "\ SUB_HOURLY_POST is NOT available with Inline Post yet." fi - -elif [ "$WRITE_DOPOST" = "FALSE" ] || \ - [ "$WRITE_DOPOST" = "NO" ]; then - WRITE_DOPOST="FALSE" fi -# -#----------------------------------------------------------------------- -# -# Make sure that QUILTING is set to a valid value. -# -#----------------------------------------------------------------------- -# + + check_var_valid_value "QUILTING" "valid_vals_QUILTING" -# -# Set QUILTING to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -QUILTING=$(echo_uppercase $QUILTING) -if [ "$QUILTING" = "TRUE" ] || \ - [ "$QUILTING" = "YES" ]; then - QUILTING="TRUE" -elif [ "$QUILTING" = "FALSE" ] || \ - [ "$QUILTING" = "NO" ]; then - QUILTING="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that PRINT_ESMF is set to a valid value. -# -#----------------------------------------------------------------------- -# +QUILTING=$(boolify $QUILTING) + check_var_valid_value "PRINT_ESMF" "valid_vals_PRINT_ESMF" -# -# Set PRINT_ESMF to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -PRINT_ESMF=$(echo_uppercase $PRINT_ESMF) -if [ "${PRINT_ESMF}" = "TRUE" ] || \ - [ "${PRINT_ESMF}" = "YES" ]; then - PRINT_ESMF="TRUE" -elif [ "${PRINT_ESMF}" = "FALSE" ] || \ - [ "${PRINT_ESMF}" = "NO" ]; then - PRINT_ESMF="FALSE" -fi +PRINT_ESMF=$(boolify $PRINT_ESMF) + # #----------------------------------------------------------------------- # @@ -2612,6 +2054,7 @@ GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # variable definitions file. # #----------------------------------------------------------------------- + # print_info_msg " Creating list of default experiment variable definitions..." @@ -2707,8 +2150,9 @@ var_name = \"${var_name}\"" # # If the variable contains only one element, then it is a scalar. (It # could be a 1-element array, but for simplicity, we treat that case as -# a scalar.) In this case, we enclose its value in double quotes and -# save the result in var_value. +# a scalar.) In this case, we enclose its value in single quotes and +# save the result in var_value. No variable expansion should be +# happening from variables saved in the var_defns file. # if [ "${num_elems}" -eq 1 ]; then @@ -3141,6 +2585,7 @@ FVCOM_FILE='${FVCOM_FILE}' # #----------------------------------------------------------------------- # +NCORES_PER_NODE='${NCORES_PER_NODE}' PE_MEMBER01='${PE_MEMBER01}' # #----------------------------------------------------------------------- diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 60162aa40f..fd03d9cad5 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -95,6 +95,15 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # +# Source the file containing the function that changes all boolean +# options to TRUE or FALSE +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/boolify.sh +# +#----------------------------------------------------------------------- +# # Source the file containing the functions that will echo given strings # as uppercase or lowercase # From 3a7ff031ce6cd91828279936386d733cb0bd85f0 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Tue, 1 Feb 2022 12:56:57 -0700 Subject: [PATCH 118/203] Update NOAA-EMC/regional_workflow references to ufs-community/regional_workflow (#671) * Update all NOAA-EMC references to ufs-community * Remove user-specific UPP directory reference. --- .github/PULL_REQUEST_TEMPLATE | 2 +- docs/UsersGuide/source/Chapter3.rst | 4 ++-- update_fork.pl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 3484674176..2af60f94ed 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -24,7 +24,7 @@ Add any links to external PRs. For example: If this PR is contributing new capabilities that need to be documented, please also include updates to the RST files in the ufs-srweather-app repository (docs/UsersGuide/source) as supporting material. ## ISSUE (optional): -If this PR is resolving or referencing one or more issues, in this repository or elewhere, list them here. For example, "Fixes issue mentioned in #123" or "Related to bug in https://github.com/NOAA-EMC/other_repository/pull/63" +If this PR is resolving or referencing one or more issues, in this repository or elewhere, list them here. For example, "Fixes issue mentioned in #123" or "Related to bug in https://github.com/ufs-community/other_repository/pull/63" ## CONTRIBUTORS (optional): If others have contributed to this work aside from the PR author, list them here diff --git a/docs/UsersGuide/source/Chapter3.rst b/docs/UsersGuide/source/Chapter3.rst index b2ee6acf81..46f16c7922 100644 --- a/docs/UsersGuide/source/Chapter3.rst +++ b/docs/UsersGuide/source/Chapter3.rst @@ -5,7 +5,7 @@ Configuring the FV3SAR Workflow *************************************** The following steps describe how to create a user-specific configuration -file in order to un your experiment in a given environment. +file in order to run your experiment in a given environment. 1. Create a user-specific configuration file named ``config.sh`` in the subdirectory ``ush`` under the ``$BASEDIR/regional_workflow`` directory containing appropriate @@ -46,7 +46,7 @@ file in order to un your experiment in a given environment. # BASEDIR="/path/to/directory/of/regional_workflow/and/NEMSfv3gfs/clones" TMPDIR="/path/to/temporary/work/directories" - UPPDIR="/scratch3/BMC/det/beck/FV3-CAM/EMC_post/sorc/ncep_post.fd" + UPPDIR="/path/to/UPP/executable/directory" CCPP="false" # CDATE="2018060400" diff --git a/update_fork.pl b/update_fork.pl index 29dec44db5..17cb67dc16 100755 --- a/update_fork.pl +++ b/update_fork.pl @@ -17,7 +17,7 @@ # 4. If all went well, you should see one of two different messages at the end: # - If your fork is already up-to-date, you should see "Already up-to-date." # - If your fork is not up-to-date, this script initiates a fast-forward merge to bring your fork -# up to date with the develop of the main repository (https://github.com/NOAA-EMC/regional_workflow). +# up to date with the develop of the main repository (https://github.com/ufs-community/regional_workflow). # Near the end git will print a line of statistics describing what changed, which will look # something like this: # @@ -55,7 +55,7 @@ } print "Username = $username\n"; -my $main_repo = "https://$username\@github.com/NOAA-EMC/regional_workflow.git"; +my $main_repo = "https://$username\@github.com/ufs-community/regional_workflow.git"; my $fork = "https://$username\@github.com/$username/regional_workflow.git"; # Set main repository as a remote repository named "upstream", per standard git conventions From c6d9937707ec8d5839b5045d0b00e18c84a3cd47 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Fri, 4 Feb 2022 09:48:44 -0700 Subject: [PATCH 119/203] Fix workflow on Cheyenne (#672) ## DESCRIPTION OF CHANGES: A couple of fixes to get the workflow running on Cheyenne. - Remove `module purge` from load_modules_run_task.sh. This no longer causes failures on Cheyenne due to intervening PR #650, but it should be removed anyway as it can cause future issues - Fixing the number of processors used in the mpirun command for the weather model on Cheyenne. I am honestly not sure how this was ever working, but this change fixes nearly all of the runtime failures currently seen on Cheyenne. ## TESTS CONDUCTED: ### Cheyenne Ran a set of WE2E tests on Cheyenne, chosen mostly at random to save core hours (I did ensure that a variety of domains were run so that several different MPI layouts were tested). Most tasks succeed, and all failures (aside from one walltime issue) are also tests that fail on Hera with the current develop branch. See issue #673 for more details. **Successful tests:** - grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 - grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR - grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta **Unsuccessful tests:** - All gfdlmp tests (grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp, grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional, grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp) - grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16 - GST_release_public_v1 - Hit walltime limit ### Hera, Jet, and Orion Ran the same set of tests on Hera, Jet, and Orion, with similar results. On Hera the GST successfully completed (though was close to reaching the walltime limit). On Jet, a few tests (grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR, grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR, grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta) failed due to missing initial and/or lateral boundary conditions. On Orion, even more tests failed due to missing ICs and LBCs (grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR, grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR, grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp, grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16). **To summarize, the only test failures were those that were also seen in develop, and mostly due to missing input files on those platforms.** ## DEPENDENCIES: This will need to be merged prior to https://github.com/ufs-community/ufs-srweather-app/pull/206 ## ISSUE: #663 has technically already been resolved, but this will fully address that specific issue. --- ush/load_modules_run_task.sh | 2 -- ush/machine/cheyenne.sh | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 59e3e156f8..28b0f56471 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -132,8 +132,6 @@ jjob_fp="$2" #----------------------------------------------------------------------- # -module purge - machine=$(echo_lowercase $MACHINE) env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" module use "${SR_WX_APP_TOP_DIR}/env" diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index d32b1f90da..a707901dbe 100755 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -56,7 +56,7 @@ FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} RUN_CMD_SERIAL="time" RUN_CMD_UTILS='mpirun -np $nprocs' -RUN_CMD_FCST='mpirun -np $nprocs' +RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' RUN_CMD_POST='mpirun -np $nprocs' # MET Installation Locations From 1762dfbcc12bcbf844c908aa9c038db8c97fecda Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 15 Feb 2022 10:16:13 -0600 Subject: [PATCH 120/203] Adding hooks for running automated WE2E tests. (#676) --- tests/WE2E/machine_suites/hera.txt | 1 + tests/WE2E/machine_suites/jet.txt | 1 + tests/WE2E/run_WE2E_tests.sh | 14 ++++- tests/WE2E/setup_WE2E_tests.sh | 91 ++++++++++++++++++++++++++++++ ush/config_defaults.sh | 4 ++ ush/setup.sh | 2 +- 6 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 tests/WE2E/machine_suites/hera.txt create mode 100644 tests/WE2E/machine_suites/jet.txt create mode 100755 tests/WE2E/setup_WE2E_tests.sh diff --git a/tests/WE2E/machine_suites/hera.txt b/tests/WE2E/machine_suites/hera.txt new file mode 100644 index 0000000000..35b26e9b1b --- /dev/null +++ b/tests/WE2E/machine_suites/hera.txt @@ -0,0 +1 @@ +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR diff --git a/tests/WE2E/machine_suites/jet.txt b/tests/WE2E/machine_suites/jet.txt new file mode 100644 index 0000000000..35b26e9b1b --- /dev/null +++ b/tests/WE2E/machine_suites/jet.txt @@ -0,0 +1 @@ +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 776c80d280..f15fb9efa8 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -86,6 +86,7 @@ Usage: machine=\"...\" \\ account=\"...\" \\ [expt_basedir=\"...\"] \\ + [exec_subdir=\"...\"] \\ [use_cron_to_relaunch=\"...\"] \\ [cron_relaunch_intvl_mnts=\"...\"] \\ [verbose=\"...\"] \\ @@ -129,6 +130,11 @@ group the WE2E tests into subdirectories for convenience, e.g. a set of tests under subdirectory testset1, another set of tests under testset2, etc. +exec_subdir: +Optional. Argument is used to set the EXEC_SUBDIR configuration +variable. Please see the ush/default_configs.sh file for a full +description. + use_cron_to_relaunch: Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH in the experiment configuration files of all the WE2E tests the user wants @@ -227,6 +233,7 @@ valid_args=( \ "machine" \ "account" \ "expt_basedir" \ + "exec_subdir" \ "use_cron_to_relaunch" \ "cron_relaunch_intvl_mnts" \ "verbose" \ @@ -687,7 +694,12 @@ Please correct and rerun." MACHINE=\"${MACHINE}\" ACCOUNT=\"${ACCOUNT}\"" - if [ ! -z "${EXPT_BASEDIR}" ]; then + if [ -n "${exec_subdir}" ]; then + expt_config_str=${expt_config_str}" +EXEC_SUBDIR=\"${exec_subdir}\"" + fi + + if [ -n "${EXPT_BASEDIR}" ]; then expt_config_str=${expt_config_str}" EXPT_BASEDIR=\"${EXPT_BASEDIR}\"" fi diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh new file mode 100755 index 0000000000..1aad100549 --- /dev/null +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -0,0 +1,91 @@ +#!/bin/bash -l + +#---------------------------------------------------------------------- +# Wrapper for the automation of UFS Short Range Weather App Workflow +# End to End Tests. +# +# The wrapper loads the appropriate workflow environment for the +# machine, and sets the machine test suite file before invoking the +# run_WE2E_tests.sh. +# +# The script is dependent on a successful build of this repo using the +# test/build.sh script in the ufs-srweather-app repository. The UFS +# build must be completed in a particular manner for this script to +# function properly, notably the location of the build and bin +# directories: +# BUILD_DIR=${APP_DIR}/build_${compiler} +# BIN_DIR=${APP_DIR}/bin_${compiler} +# +# Example: ./end_to_end_tests.sh hera zrtrr +#---------------------------------------------------------------------- + +#----------------------------------------------------------------------- +# Set variables +#----------------------------------------------------------------------- + +function usage { + echo + echo "Usage: $0 machine slurm_account | -h" + echo + echo " machine [required] is one of: ${machines[@]}" + echo " slurm_account [required] case sensitive name of the user-specific slurm account" + echo " -h display this help" + echo + exit 1 + +} + +machines=( hera jet ) + +if [ "$1" = "-h" ] ; then usage ; fi +[[ $# -le 1 ]] && usage + +machine=$1 +machine=$(echo "${machine}" | tr '[A-Z]' '[a-z]') # scripts in sorc need lower case machine name + +account=$2 + +#----------------------------------------------------------------------- +# Set directories +#----------------------------------------------------------------------- +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) + +TESTS_DIR=$( dirname "${scrfunc_dir}" ) +REGIONAL_WORKFLOW_DIR=$( dirname "${TESTS_DIR}" ) +SRW_APP_DIR=$( dirname "${REGIONAL_WORKFLOW_DIR}" ) +TOP_DIR=$( dirname "${SRW_APP_DIR}" ) + +EXPTS_DIR=${TOP_DIR}/expt_dirs + +#----------------------------------------------------------------------- +# Set the path to the machine-specific test suite file. +#----------------------------------------------------------------------- + +auto_file=${scrfunc_dir}/machine_suites/${machine}.txt + +#---------------------------------------------------------------------- +# Use exec_subdir consistent with the automated build. +#---------------------------------------------------------------------- + +exec_subdir='bin_intel/bin' + +#----------------------------------------------------------------------- +# Run E2E Tests +#----------------------------------------------------------------------- + +# Load Python Modules +env_file="${SRW_APP_DIR}/env/wflow_${machine}.env" +source ${env_file} +echo "-- Load environment =>" $env_file + +module list + +# Run the E2E Workflow tests +./run_WE2E_tests.sh \ + tests_file=${auto_file} \ + machine=${machine} \ + account=${account} \ + exec_subdir=${exec_subdir} + diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index e7860e021e..6de917fa21 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -218,10 +218,14 @@ CRON_RELAUNCH_INTVL_MNTS="03" # # dir_doc_end # +# EXEC_SUBDIR: +# The name of the subdirectory of ufs-srweather-app where executables are +# installed. #----------------------------------------------------------------------- # EXPT_BASEDIR="" EXPT_SUBDIR="" +EXEC_SUBDIR="bin" # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index 74426ef7d8..2cbd0f6fc0 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -401,7 +401,7 @@ SORCDIR="$HOMErrfs/sorc" SRC_DIR="${SR_WX_APP_TOP_DIR}/src" PARMDIR="$HOMErrfs/parm" MODULES_DIR="$HOMErrfs/modulefiles" -EXECDIR="${SR_WX_APP_TOP_DIR}/bin" +EXECDIR="${SR_WX_APP_TOP_DIR}/${EXEC_SUBDIR}" TEMPLATE_DIR="$USHDIR/templates" VX_CONFIG_DIR="$TEMPLATE_DIR/parm" METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" From 6c17af9986a92e3c1ee12a426f31e8cc0a46f4eb Mon Sep 17 00:00:00 2001 From: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Date: Sat, 19 Feb 2022 16:43:30 -0700 Subject: [PATCH 121/203] Modify exregional_run_post to use the new fortran namelist (itag) file (#677) Co-authored-by: Tracy --- scripts/exregional_run_post.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index a9b07efef9..631b502174 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -196,7 +196,7 @@ tmmark="tm00" # #----------------------------------------------------------------------- # -# Create a text file (itag) containing arguments to pass to the post- +# Create the namelist file (itag) containing arguments to pass to the post- # processor's executable. # #----------------------------------------------------------------------- @@ -236,7 +236,7 @@ post_dd=${post_time:6:2} post_hh=${post_time:8:2} post_mn=${post_time:10:2} # -# Create the input text file to the post-processor executable. +# Create the input namelist file to the post-processor executable. # if [ ${FCST_MODEL} = "fv3gfs_aqm" ]; then post_itag_add="aqfcmaq_on=.true.," @@ -244,12 +244,13 @@ else post_itag_add="" fi cat > itag < Date: Tue, 22 Feb 2022 16:45:54 -0700 Subject: [PATCH 122/203] Update CODEOWNERS file (#680) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c05ae70791..154c5ba16d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA +* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @BenjaminBlake-NOAA @RatkoVasic-NOAA @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @jwolff-ncar @dmwright526 # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners From 2b3df16b50705198c138313694f0081b320b00c3 Mon Sep 17 00:00:00 2001 From: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Date: Thu, 24 Feb 2022 15:57:43 -0700 Subject: [PATCH 123/203] Enhancement for the ability to output satellite products from the UPP using the CRTM (#682) Co-authored-by: Tracy --- scripts/exregional_run_post.sh | 18 ++++++++++++++++++ ush/config_defaults.sh | 22 ++++++++++++++++++++++ ush/setup.sh | 16 ++++++++++++++++ ush/valid_param_vals.sh | 1 + 4 files changed, 57 insertions(+) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 631b502174..e45223ce92 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -171,6 +171,24 @@ temporary work directory (tmp_dir): fi cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . +if [ ${USE_CRTM} = "TRUE" ]; then + cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Water/Big_Endian/Nalli.IRwater.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/MW_Water/Big_Endian/FAST*.bin ./ + cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Land/SEcategory/Big_Endian/NPOESS.IRland.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Snow/SEcategory/Big_Endian/NPOESS.IRsnow.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Ice/SEcategory/Big_Endian/NPOESS.IRice.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/AerosolCoeff/Big_Endian/AerosolCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/CloudCoeff/Big_Endian/CloudCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/fix/SpcCoeff/Big_Endian/*.bin ./ + cp_vrfy ${CRTM_DIR}/fix/TauCoeff/ODPS/Big_Endian/*.bin ./ + print_info_msg " +==================================================================== +Copying the external CRTM fix files from CRTM_DIR to the temporary +work directory (tmp_dir): + CRTM_DIR = \"${CRTM_DIR}\" + tmp_dir = \"${tmp_dir}\" +====================================================================" +fi # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 6de917fa21..1e59038ee5 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1732,6 +1732,28 @@ CUSTOM_POST_CONFIG_FP="" # #----------------------------------------------------------------------- # +# Set parameters associated with outputting satellite fields in the UPP +# grib2 files using the Community Radiative Transfer Model (CRTM). +# +# USE_CRTM: +# Flag that defines whether external CRTM coefficient files have been +# staged by the user in order to output synthetic statellite products +# available within the UPP. If this is set to "TRUE", then the workflow +# will check for these files in the directory CRTM_DIR. Otherwise, it is +# assumed that no satellite fields are being requested in the UPP +# configuration. +# +# CRTM_DIR: +# This is the path to the top CRTM fix file directory. This is only used +# if USE_CRTM is set to "TRUE". +# +#----------------------------------------------------------------------- +# +USE_CRTM="FALSE" +CRTM_DIR="" +# +#----------------------------------------------------------------------- +# # Set parameters associated with running ensembles. Definitions: # # DO_ENSEMBLE: diff --git a/ush/setup.sh b/ush/setup.sh index 2cbd0f6fc0..407d20d5c2 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -754,6 +754,22 @@ fi # #----------------------------------------------------------------------- # +# If using external CRTM fix files to allow post-processing of synthetic +# satellite products from the UPP, then make sure the fix file directory +# exists. +# +#----------------------------------------------------------------------- +# +if [ ${USE_CRTM} = "TRUE" ]; then + if [ ! -d "${CRTM_DIR}" ]; then + print_err_msg_exit " +The external CRTM fix file directory specified by CRTM_DIR does not exist: + CRTM_DIR = \"${CRTM_DIR}\"" + fi +fi +# +#----------------------------------------------------------------------- +# # The forecast length (in integer hours) cannot contain more than 3 cha- # racters. Thus, its maximum value is 999. Check whether the specified # forecast length exceeds this maximum value. If so, print out a warn- diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index c421441309..e80721d51e 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -68,6 +68,7 @@ valid_vals_NOMADS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_NOMADS_file_type=("GRIB2" "grib2" "NEMSIO" "nemsio") valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CUSTOM_POST_CONFIG_FILE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_USE_CRTM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From ddfc256db7e9023a37c1638c4490fde7aec001c2 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 28 Feb 2022 17:04:42 -0700 Subject: [PATCH 124/203] Use non-login shell for `launch_FV3LAM_wflow.sh`; remove support for `WCOSS_CRAY`; fix cron capability for `tcsh` users on Cheyenne (#675) * Remove unneeded sourcing of source_util_funcs.sh; add "%s" to printf calls since that's the proper calling method; edit comments. * Generalize machine files. Details: * Add a wrapper (source_machine_file.sh) for sourcing the machine file that allows other commands common to all machines to be called. * Change the scalar variable MODULE_INIT_PATH in the machine files to the array variable ENV_INIT_SCRIPTS_FPS that specifies the list of system scripts that need to be sourced (e.g. to make the "module" command available in a given script). This is needed because on Cheyenne, at least two system scripts need to be sourced (to enable "module" and "qsub"). * Move the "ulimit" commands at the ends of the machine files into the new variable PRE_TASK_CMDS so that they are not called every time the machine file is sourced. They will be called only if a given script issues an "eval ${PRE_TASK_CMDS}" (which all the ex-scripts will do). * In the relevant ex-scripts: (1) Change sourcing of machine files to use the wrapper source_machine_file.sh; (2) Use "eval" to evaluate the contents of PRE_TASK_CMDS. * In the WE2E script, change sourcing of the machine file to use the wrapper source_machine_file.sh. * Add new variable valid_vals_BOOLEAN to constants.sh so that this file can be sourced and the valid values for a boolean can be made available to any other script. * Bug fix. * Remove file that was accidentally added in previous commit. * Change the way crontab is called so that it also works on Cheyenne (for tcsh users). Details: * Introduce new function get_crontab_contents() that takes as input whether or not the calling script is itself being called from a cron job and returns (1) the path to the appropriate crontab command and (2) the contents of the user's cron table. * Such a function is needed because on Cheyenne, the location of the crontab command is different depending on whether or not the script that's calling crontab is itself called from a cron job (because on Cheyenne, "crontab" is containerized, and that complicates things). * Use get_crontab_contents() in generate_FV3LAM_wflow.sh and launch_FV3LAM_wflow.sh (instead of simply calling "crontab" because the latter approach doesn't work on Cheyenne, at least not with users whose login shell is tcsh). * Add "called_from_cron" as an optional argument to launch_FV3LAM_wflow.sh [so that it can then be passed on to get_crontab_contents()]. This argument is only used in the cron job that relaunches the workflow (which is created only if USE_CRON_TO_RELAUNCH is set to "TRUE"). * Having an optional argument like this seems to be the best way to tell launch_FV3LAM_wflow.sh whether or not it is running from a cron job. * launch_FV3LAM_wflow.sh can still be called from the command line without any arguments (since the default value of "called_from_cron" is "FALSE"). * Generalize the way commands are initialized so that any number of system scripts can be sourced in a given script (currently, only "module" is initialized). Details: * Introduce the new function init_env() that initializes the envrionment of a script by sourcing necessary system scripts. The full paths to these system scripts are specified in the array ENV_INIT_SCRIPTS_FPS in the machine files. * This function is needed because (1) this sourcing needs to be done in a couple of different scripts in the SRW App and (2) on some machines (e.g. Cheyenne), more than one system script may need to be sourced. * Use the new init_env() function in launch_FV3LAM_wflow.sh and load_modules_run_task.sh. * In load_modules_run_task.sh, init_env() replaces sourcing of only the system script that defines the "module" command. That is because on Cheyenne, in addition to the "module" command, the "qsub" command needs to be defined/initialized (by sourcing a second system script named pbs.sh). * Replace calls to "crontab -l" by echoing of already obtained contents. Fix comments and informational messages. * For Cheyenne, don't need to source two separate system scripts. Just sourcing "/etc/profile" is enough to make both the "module" and "qsub" commands (and probably all other system-supported commands) available in non-login scripts. * Make script exit with an error message if rocoto commands fail. * Fix the system script that needs to be sourced on Hera to get "module" (and other commands) to work. * In init_env.sh, declare "local" variables and change the index of the for-loop so it's different than the variable i used (and unset) by the system script on Hera. * Fix the system script that needs to be sourced on Orion to enable the "module" and other commands in a non-login bash shell. * Fix the system script on Jet that needs to be sourced to enable the "module" and other commands in a non-login bash shell. * Update comments. * Bug fix: Make sure the variable __crontab_cmd__ is defined for WCOSS_DELL_P3. * Try changing the system script to source on WCOSS_DELL_P3 to "/etc/profile" (since it works on the other machines to enable the "module" and other commands). This needs to be tested by someone who has access to WCOSS_DELL_P3. * Changes to try to make the machine file work for WCOSS_DELL_P3. Not yet tested. * Fix modulepath issue on wcoss * Fix issues on wcoss cray * Fix crontab issue on wcoss cray * Remove support for WCOSS_CRAY. * Place double qoutes around ${RUN_CMD_...} in if-statements that check whether the RUN_CMD_... variable is empty, i.e. -z "${RUN_CMD_...}". This is needed because on Cheyenne, not having the double quotes generates an error when RUN_CMD_... consists of a command that contains spaces (e.g. "mpirun -np ..."). Co-authored-by: chan-hoo --- scripts/exregional_make_grid.sh | 32 +------- scripts/exregional_make_ics.sh | 21 +---- scripts/exregional_make_lbcs.sh | 21 +---- scripts/exregional_make_orog.sh | 29 +------ scripts/exregional_make_sfc_climo.sh | 26 +------ scripts/exregional_run_fcst.sh | 28 +------ scripts/exregional_run_post.sh | 37 +-------- tests/WE2E/run_WE2E_tests.sh | 2 +- ush/constants.sh | 8 ++ ush/generate_FV3LAM_wflow.sh | 56 +++++++------- ush/get_crontab_contents.sh | 83 ++++++++++++++++++++ ush/init_env.sh | 51 +++++++++++++ ush/launch_FV3LAM_wflow.sh | 110 +++++++++++++++++++-------- ush/load_modules_run_task.sh | 56 ++------------ ush/machine/cheyenne.sh | 16 ++-- ush/machine/hera.sh | 17 ++--- ush/machine/jet.sh | 17 ++--- ush/machine/odin.sh | 14 ++-- ush/machine/orion.sh | 16 ++-- ush/machine/stampede.sh | 14 ++-- ush/machine/wcoss_cray.sh | 77 ------------------- ush/machine/wcoss_dell_p3.sh | 21 +++-- ush/setup.sh | 4 +- ush/source_machine_file.sh | 17 +++++ ush/templates/FV3LAM_wflow.xml | 19 +---- ush/valid_param_vals.sh | 2 +- 26 files changed, 364 insertions(+), 430 deletions(-) create mode 100644 ush/get_crontab_contents.sh create mode 100644 ush/init_env.sh delete mode 100755 ush/machine/wcoss_cray.sh create mode 100644 ush/source_machine_file.sh diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index ebc580a4da..742770630b 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -83,36 +83,10 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -case "$MACHINE" in +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} - "WCOSS_CRAY") - { save_shell_opts; set +x; } > /dev/null 2>&1 - . $MODULESHOME/init/sh - module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 - module list - { restore_shell_opts; } > /dev/null 2>&1 - export NODES=1 - export RUN_CMD_SERIAL="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" - export KMP_AFFINITY=disabled - ulimit -s unlimited - ulimit -a - ;; - - "WCOSS_DELL_P3") - { save_shell_opts; set +x; } > /dev/null 2>&1 - module list - { restore_shell_opts; } > /dev/null 2>&1 - export RUN_CMD_SERIAL="mpirun" - ulimit -s unlimited - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac - -if [ -z ${RUN_CMD_SERIAL:-} ] ; then +if [ -z "${RUN_CMD_SERIAL:-}" ] ; then print_err_msg_exit " \ Run command was not set in machine file. \ Please set RUN_CMD_SERIAL for your platform" diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 7161f16aab..ca27553977 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -86,27 +86,12 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} # #----------------------------------------------------------------------- # -case "$MACHINE" in - - "WCOSS_CRAY") - ulimit -s unlimited - RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth" - ;; - - "WCOSS_DELL_P3") - ulimit -s unlimited - RUN_CMD_UTILS="mpirun" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) -if [ -z ${RUN_CMD_UTILS:-} ] ; then +if [ -z "${RUN_CMD_UTILS:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f123dfc408..23acdc1f14 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -86,27 +86,12 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} # #----------------------------------------------------------------------- # -case "$MACHINE" in - - "WCOSS_CRAY") - ulimit -s unlimited - RUN_CMD_UTILS="aprun -b -j1 -n48 -N12 -d1 -cc depth" - ;; - - "WCOSS_DELL_P3") - ulimit -s unlimited - RUN_CMD_UTILS="mpirun" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS )) -if [ -z ${RUN_CMD_UTILS:-} ] ; then +if [ -z "${RUN_CMD_UTILS:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 430aa6a8d4..1a4d528f24 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -95,33 +95,10 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # -case "$MACHINE" in +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} - "WCOSS_CRAY") - { save_shell_opts; set +x; } > /dev/null 2>&1 - . $MODULESHOME/init/sh - module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 - module list - { restore_shell_opts; } > /dev/null 2>&1 - NODES=1 - RUN_CMD_SERIAL="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" - ulimit -s unlimited - ulimit -a - ;; - - "WCOSS_DELL_P3") - ulimit -s unlimited - ulimit -a - RUN_CMD_SERIAL="mpirun" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac - -if [ -z ${RUN_CMD_SERIAL:-} ] ; then +if [ -z "${RUN_CMD_SERIAL:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_SERIAL for your platform" diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index ada4929878..129cfcb75b 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -136,32 +136,12 @@ EOF # #----------------------------------------------------------------------- # -case "$MACHINE" in - - "WCOSS_CRAY") - RUN_CMD_UTILS=${APRUN:-"aprun -j 1 -n 6 -N 6"} - ;; - - "WCOSS_DELL_P3") -# Specify computational resources. - export NODES=2 - export ntasks=48 - export ptile=24 - export threads=1 - export MP_LABELIO=yes - export OMP_NUM_THREADS=$threads - RUN_CMD_UTILS="mpirun" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) -if [ -z ${RUN_CMD_UTILS:-} ] ; then +if [ -z "${RUN_CMD_UTILS:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 10e1d6240c..d7c3d48f62 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -99,34 +99,12 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} # #----------------------------------------------------------------------- # -case "$MACHINE" in - - "WCOSS_CRAY") - ulimit -s unlimited - ulimit -a - - if [ ${PE_MEMBER01} -gt 24 ];then - RUN_CMD_FCST="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" - else - RUN_CMD_FCST="aprun -b -j1 -n${PE_MEMBER01} -N${PE_MEMBER01} -d1 -cc depth" - fi - ;; - - "WCOSS_DELL_P3") - ulimit -s unlimited - ulimit -a - RUN_CMD_FCST="mpirun -l -np ${PE_MEMBER01}" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) -if [ -z ${RUN_CMD_FCST:-} ] ; then +if [ -z "${RUN_CMD_FCST:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_FCST for your platform" diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index e45223ce92..a2aea08d2c 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -92,42 +92,11 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} # #----------------------------------------------------------------------- # -case "$MACHINE" in - - "WCOSS_CRAY") - -# Specify computational resources. - export NODES=2 - export ntasks=48 - export ptile=24 - export threads=1 - export MP_LABELIO=yes - export OMP_NUM_THREADS=$threads - - RUN_CMD_POST="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" - ;; - - "WCOSS_DELL_P3") - -# Specify computational resources. - export NODES=2 - export ntasks=48 - export ptile=24 - export threads=1 - export MP_LABELIO=yes - export OMP_NUM_THREADS=$threads - - RUN_CMD_POST="mpirun" - ;; - - *) - source ${MACHINE_FILE} - ;; - -esac +source $USHDIR/source_machine_file.sh +eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) -if [ -z ${RUN_CMD_POST:-} ] ; then +if [ -z "${RUN_CMD_POST:-}" ] ; then print_err_msg_exit "\ Run command was not set in machine file. \ Please set RUN_CMD_POST for your platform" diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index f15fb9efa8..bfab3128a5 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -682,7 +682,7 @@ Please correct and rerun." # Set the machine-specific configuration settings by sourcing the # machine file in the ush directory - source $MACHINE_FILE + source $ushdir/source_machine_file.sh expt_config_str=${expt_config_str}"\ # diff --git a/ush/constants.sh b/ush/constants.sh index fb19e61f84..0d68019aed 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -15,3 +15,11 @@ degs_per_radian=$( bc -l <<< "360.0/(2.0*$pi_geom)" ) # Radius of the Earth in meters. radius_Earth="6371200.0" +# +#----------------------------------------------------------------------- +# +# Other. +# +#----------------------------------------------------------------------- +# +valid_vals_BOOLEAN=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 4776c3d455..de466fb1be 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -59,6 +59,7 @@ ushdir="${scrfunc_dir}" #----------------------------------------------------------------------- # . $ushdir/source_util_funcs.sh +. $ushdir/get_crontab_contents.sh . $ushdir/set_FV3nml_sfc_climo_filenames.sh # #----------------------------------------------------------------------- @@ -500,11 +501,12 @@ if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then print_info_msg "$VERBOSE" " Copying contents of user cron table to backup file: crontab_backup_fp = \"${crontab_backup_fp}\"" - if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - cp_vrfy "/u/$USER/cron/mycrontab" "${crontab_backup_fp}" - else - crontab -l > ${crontab_backup_fp} - fi + + called_from_cron=${called_from_cron:-"FALSE"} + get_crontab_contents called_from_cron=${called_from_cron} \ + outvarname_crontab_cmd="crontab_cmd" \ + outvarname_crontab_contents="crontab_contents" + echo "${crontab_contents}" > "${crontab_backup_fp}" # # Below, we use "grep" to determine whether the crontab line that the # variable CRONTAB_LINE contains is already present in the cron table. @@ -514,23 +516,23 @@ Copying contents of user cron table to backup file: crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ $SED -r -e "s%[*]%\\\\*%g" ) # -# In the grep command below, the "^" at the beginning of the string be- -# ing passed to grep is a start-of-line anchor while the "$" at the end -# of the string is an end-of-line anchor. Thus, in order for grep to -# find a match on any given line of the output of "crontab -l", that -# line must contain exactly the string in the variable crontab_line_- -# esc_astr without any leading or trailing characters. This is to eli- -# minate situations in which a line in the output of "crontab -l" con- -# tains the string in crontab_line_esc_astr but is precedeeded, for ex- -# ample, by the comment character "#" (in which case cron ignores that -# line) and/or is followed by further commands that are not part of the -# string in crontab_line_esc_astr (in which case it does something more -# than the command portion of the string in crontab_line_esc_astr does). -# - if [ "$MACHINE" = "WCOSS_DELL_P3" ];then +# In the grep command below, the "^" at the beginning of the string +# passed to grep is a start-of-line anchor, and the "$" at the end is +# an end-of-line anchor. Thus, in order for grep to find a match on +# any given line of the cron table's contents, that line must contain +# exactly the string in the variable crontab_line_esc_astr without any +# leading or trailing characters. This is to eliminate situations in +# which a line in the cron table contains the string in crontab_line_esc_astr +# but is precedeeded, for example, by the comment character "#" (in which +# case cron ignores that line) and/or is followed by further commands +# that are not part of the string in crontab_line_esc_astr (in which +# case it does something more than the command portion of the string in +# crontab_line_esc_astr does). +# + if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then grep_output=$( grep "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" ) else - grep_output=$( crontab -l | grep "^${crontab_line_esc_astr}$" ) + grep_output=$( echo "${crontab_contents}" | grep "^${crontab_line_esc_astr}$" ) fi exit_status=$? @@ -548,10 +550,10 @@ Adding the following line to the user's cron table in order to automatically resubmit SRW workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" - if [ "$MACHINE" = "WCOSS_DELL_P3" ];then + if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then echo "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" else - ( crontab -l; echo "${CRONTAB_LINE}" ) | crontab - + ( echo "${crontab_contents}"; echo "${CRONTAB_LINE}" ) | ${crontab_cmd} fi fi @@ -912,8 +914,8 @@ cp_vrfy $USHDIR/${EXPT_CONFIG_FN} $EXPTDIR # # For convenience, print out the commands that need to be issued on the # command line in order to launch the workflow and to check its status. -# Also, print out the command that should be placed in the user's cron- -# tab in order for the workflow to be continually resubmitted. +# Also, print out the line that should be placed in the user's cron table +# in order for the workflow to be continually resubmitted. # #----------------------------------------------------------------------- # @@ -978,14 +980,14 @@ Note that: task(s) to the queue. 2) In order for the output of the rocotostat command to be up-to-date, - the rocotorun command must be issued immediately before the rocoto- - stat command. + the rocotorun command must be issued immediately before issuing the + rocotostat command. For automatic resubmission of the workflow (say every 3 minutes), the following line can be added to the user's crontab (use \"crontab -e\" to edit the cron table): -*/3 * * * * cd $EXPTDIR && ./launch_FV3LAM_wflow.sh +*/3 * * * * cd $EXPTDIR && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" " fi diff --git a/ush/get_crontab_contents.sh b/ush/get_crontab_contents.sh new file mode 100644 index 0000000000..dbf683790b --- /dev/null +++ b/ush/get_crontab_contents.sh @@ -0,0 +1,83 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that returns the contents of the user's +# cron table as well as the command to use to manipulate the cron table +# (i.e. the "crontab" command, but on some platforms the version or +# location of this may change depending on other circumstances, e.g. on +# Cheyenne, this depends on whether a script that wants to call "crontab" +# is itself being called from a cron job). Arguments are as follows: +# +# called_from_cron: +# Boolean flag that specifies whether this function (and the scripts or +# functions that are calling it) are called as part of a cron job. Must +# be set to "TRUE" or "FALSE". +# +# outvarname_crontab_cmd: +# Name of the output variable that will contain the command to issue for +# the system "crontab" command. +# +# outvarname_crontab_contents: +# Name of the output variable that will contain the contents of the +# user's cron table. +# +#----------------------------------------------------------------------- +# +function get_crontab_contents() { + + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + + local valid_args=( \ + "called_from_cron" \ + "outvarname_crontab_cmd" \ + "outvarname_crontab_contents" \ + ) + process_args valid_args "$@" + print_input_args "valid_args" + + local __crontab_cmd__ \ + __crontab_contents__ + # + # Make sure called_from_cron is set to a valid value. + # + source $USHDIR/constants.sh + check_var_valid_value "called_from_cron" "valid_vals_BOOLEAN" + called_from_cron=$( boolify ${called_from_cron} ) + + if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + __crontab_cmd__="" + __crontab_contents__=$( cat "/u/$USER/cron/mycrontab" ) + else + __crontab_cmd__="crontab" + # + # On Cheyenne, simply typing "crontab" will launch the crontab command + # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized + # version of crontab that will work if called from scripts that are + # themselves being called as cron jobs. In that case, we must instead + # call the system version of crontab at /usr/bin/crontab. + # + if [ "$MACHINE" = "CHEYENNE" ]; then + if [ -n "${called_from_cron}" ] && [ "${called_from_cron}" = "TRUE" ]; then + __crontab_cmd__="/usr/bin/crontab" + fi + fi + __crontab_contents__=$( ${__crontab_cmd__} -l ) + fi + # + # On Cheyenne, the output of the "crontab -l" command contains a 3-line + # header (comments) at the top that is not actually part of the user's + # cron table. This needs to be removed to avoid adding an unnecessary + # copy of this header to the user's cron table. + # + if [ "$MACHINE" = "CHEYENNE" ]; then + __crontab_contents__=$( echo "${__crontab_contents__}" | tail -n +4 ) + fi + # + # Set output variables. + # + printf -v ${outvarname_crontab_cmd} "${__crontab_cmd__}" + printf -v ${outvarname_crontab_contents} "${__crontab_contents__}" + + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/init_env.sh b/ush/init_env.sh new file mode 100644 index 0000000000..d883766325 --- /dev/null +++ b/ush/init_env.sh @@ -0,0 +1,51 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that sources scripts (usually system +# scripts) to initialize various commands in the environment, e.g. the +# "module" command. The full paths to these scripts are specified in +# the machine files in the array ENV_INIT_SCRIPTS_FPS. +# +# env_init_scripts_fps: +# Full paths to the system scripts to source. +# +#----------------------------------------------------------------------- +# +function init_env() { + + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + + local valid_args=( \ + "env_init_scripts_fps" \ + ) + process_args valid_args "$@" + print_input_args "valid_args" + + local num_scripts \ + n \ + fp + + num_scripts="${#env_init_scripts_fps[@]}" + for (( n=0; n<${num_scripts}; n++ )); do + fp="${env_init_scripts_fps[$n]}" + print_info_msg "$DEBUG" "\ +Attempting to source script: + fp = \"$fp\"" + if [ -f "$fp" ]; then + # The scripts being sourced here may have undefined variables, but since + # they are system scripts outside of the SRW App, they cannot be changed. + # Thus, we allow for undefined variables by temporarily using "set +u". + set +u + source "$fp" && print_info_msg "$DEBUG" "Succeeded." + set -u + else + print_err_msg_exit "\ +The script to source does not exist or is not a regular file: + fp = \"$fp\"" + fi + + done + + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 4c881be2db..ddee812fa7 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -1,4 +1,4 @@ -#!/bin/bash -l +#!/bin/bash # #----------------------------------------------------------------------- @@ -31,9 +31,6 @@ else fi scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) - -ushdir="${scrfunc_dir}" -. $ushdir/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -93,12 +90,48 @@ fi # #----------------------------------------------------------------------- # -# Source the variable definitions file for the experiment. +# Source necessary files. # #----------------------------------------------------------------------- # . $exptdir/var_defns.sh -. ${USHDIR}/source_util_funcs.sh +. $USHDIR/source_machine_file.sh +. $USHDIR/constants.sh +. $USHDIR/source_util_funcs.sh +. $USHDIR/init_env.sh +. $USHDIR/get_crontab_contents.sh +# +#----------------------------------------------------------------------- +# +# Declare arguments. +# +#----------------------------------------------------------------------- +# +valid_args=( \ + "called_from_cron" \ + ) +process_args valid_args "$@" +print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Make sure called_from_cron is set to a valid value. +# +#----------------------------------------------------------------------- +# +called_from_cron=${called_from_cron:-"FALSE"} +check_var_valid_value "called_from_cron" "valid_vals_BOOLEAN" +called_from_cron=$(boolify ${called_from_cron}) +# +#----------------------------------------------------------------------- +# +# Initialize the environment, e.g. by making the "module" command as well +# as others available. +# +#----------------------------------------------------------------------- +# +env_init_scripts_fps_str="( "$(printf "\"%s\" " "${ENV_INIT_SCRIPTS_FPS[@]}")")" +init_env env_init_scripts_fps="${env_init_scripts_fps_str}" # #----------------------------------------------------------------------- # @@ -117,7 +150,6 @@ expt_name="${EXPT_SUBDIR}" #----------------------------------------------------------------------- # env_fp="${SR_WX_APP_TOP_DIR}/env/${WFLOW_ENV_FN}" -module purge source "${env_fp}" || print_err_msg_exit "\ Sourcing platform-specific environment file (env_fp) for the workflow task failed: @@ -155,23 +187,22 @@ cd_vrfy "$exptdir" # # Issue the rocotorun command to (re)launch the next task in the workflow. # Then check for error messages in the output of rocotorun. If any are -# found, it means the end-to-end run of the workflow failed. In this -# case, we remove the crontab entry that launches the workflow, and we -# append an appropriate failure message at the end of the launch log -# file. +# found, it means the end-to-end run of the workflow failed, so set the +# status of the workflow to "FAILURE". # #----------------------------------------------------------------------- # tmp_fn="rocotorun_output.txt" rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" -eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 +eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 || \ + print_err_msg_exit "\ +Call to \"rocotorun\" failed with return code $?." rocotorun_output=$( cat "${tmp_fn}" ) rm "${tmp_fn}" error_msg="sbatch: error: Batch job submission failed:" -# Job violates accounting/QOS policy (job submit limit, user's size and/or time limits)" while read -r line; do - grep_output=$( printf "$line" | grep "${error_msg}" ) + grep_output=$( printf "%s" "$line" | grep "${error_msg}" ) if [ $? -eq 0 ]; then wflow_status="FAILURE" break @@ -182,19 +213,20 @@ done <<< "${rocotorun_output}" # # Issue the rocotostat command to obtain a table specifying the status # of each task. Then check for dead tasks in the output of rocotostat. -# If any are found, it means the end-to-end run of the workflow failed. -# In this case, we remove the crontab entry that launches the workflow, -# and we append an appropriate failure message at the end of the launch -# log file. +# If any are found, it means the end-to-end run of the workflow failed, +# so set the status of the workflow (wflow_status) to "FAILURE". # #----------------------------------------------------------------------- # rocotostat_cmd="rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" +rocotostat_output=$( eval ${rocotostat_cmd} 2>&1 || \ + print_err_msg_exit "\ +Call to \"rocotostat\" failed with return code $?." + ) -rocotostat_output=$( eval ${rocotostat_cmd} 2>&1 ) error_msg="DEAD" while read -r line; do - grep_output=$( printf "$line" | grep "${error_msg}" ) + grep_output=$( printf "%s" "$line" | grep "${error_msg}" ) if [ $? -eq 0 ]; then wflow_status="FAILURE" break @@ -208,7 +240,7 @@ done <<< "${rocotostat_output}" # #----------------------------------------------------------------------- # -printf " +printf "%s" " ======================================================================== Start of output from script \"${scrfunc_fn}\". @@ -241,8 +273,8 @@ ${rocotostat_output} # the status of each cycle in the workflow. The output of this command # has the following format: # -# CYCLE STATE ACTIVATED DEACTIVATED -# 201905200000 Active Nov 07 2019 00:23:30 - +# CYCLE STATE ACTIVATED DEACTIVATED +# 201905200000 Active Nov 07 2019 00:23:30 - # ... # # Thus, the first row is a header line containing the column titles, and @@ -313,7 +345,7 @@ fi # #----------------------------------------------------------------------- # -printf " +printf "%s" " Summary of workflow status: ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -329,9 +361,9 @@ End of output from script \"${scrfunc_fn}\". # #----------------------------------------------------------------------- # -# If the workflow status is now either "SUCCESS" or "FAILURE", indicate -# this by appending an appropriate workflow completion message to the -# end of the launch log file. +# If the workflow status (wflow_status) has been set to either "SUCCESS" +# or "FAILURE", indicate this by appending an appropriate workflow +# completion message to the end of the launch log file. # #----------------------------------------------------------------------- # @@ -369,29 +401,41 @@ script for this experiment: crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ $SED -r -e "s%[*]%\\\\*%g" ) # +# Get the full contents of the user's cron table. +# + get_crontab_contents called_from_cron=${called_from_cron} \ + outvarname_crontab_cmd="crontab_cmd" \ + outvarname_crontab_contents="crontab_contents" +# +# Remove the line in the contents of the cron table corresponding to the +# current forecast experiment (if that line is part of the contents). +# Then record the results back into the user's cron table. +# # In the string passed to the grep command below, we use the line start # and line end anchors ("^" and "$", respectively) to ensure that we # only find lines in the crontab that contain exactly the string in # crontab_line_esc_astr without any leading or trailing characters. # - if [ "$MACHINE" = "WCOSS_DELL_P3" ];then - grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" \ - > tmpfile && mv_vrfy tmpfile "/u/$USER/cron/mycrontab" + crontab_contents=$( echo "${crontab_contents}" | grep -v "^${crontab_line_esc_astr}$" ) + + if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + echo "${crontab_contents}" > "/u/$USER/cron/mycrontab" else - ( crontab -l | grep -v "^${crontab_line_esc_astr}$" ) | crontab - + echo "${crontab_contents}" | ${crontab_cmd} fi + fi # # Print the workflow completion message to the launch log file. # - printf "$msg" >> ${WFLOW_LAUNCH_LOG_FN} 2>&1 + printf "%s" "$msg" >> ${WFLOW_LAUNCH_LOG_FN} 2>&1 # # If the stdout from this script is being sent to the screen (e.g. it is # not being redirected to a file), then also print out the workflow # completion message to the screen. # if [ -t 1 ]; then - printf "$msg" + printf "%s" "$msg" fi fi diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 28b0f56471..56c005faa3 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -3,12 +3,14 @@ # #----------------------------------------------------------------------- # -# Source the variable definitions file and the bash utility functions. +# Source necessary files. # #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_machine_file.sh . $USHDIR/source_util_funcs.sh +. $USHDIR/init_env.sh # #----------------------------------------------------------------------- # @@ -65,56 +67,13 @@ fi # #----------------------------------------------------------------------- # -# Source the script that initializes the Lmod (Lua-based module) system/ -# software for handling modules. This script defines the module() and -# other functions. These are needed so we can perform the "module use -# ..." and "module load ..." calls later below that are used to load the -# appropriate module file for the specified task. +# Initialize the environment, e.g. by making available the "module" +# command as well as others. # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " -Initializing the shell function \"module()\" (and others) in order to be -able to use \"module load ...\" to load necessary modules ..." - -case "$MACHINE" in -# - "WCOSS_CRAY") - . /opt/modules/default/init/sh - ;; -# - "WCOSS_DELL_P3") - . /usrx/local/prod/lmod/lmod/init/sh - ;; -# - "HERA") - . /apps/lmod/lmod/init/sh - ;; -# - "ORION") - . /apps/lmod/lmod/init/sh - ;; -# - "JET") - . /apps/lmod/lmod/init/sh - ;; -# - "CHEYENNE") - . /glade/u/apps/ch/opt/lmod/8.1.7/lmod/8.1.7/init/sh - ;; -# - *) - if [[ -n ${LMOD_PATH:-""} && -f ${LMOD_PATH:-""} ]] ; then - . ${LMOD_PATH} - else - print_err_msg_exit "\ - The script to source to initialize lmod (module loads) has not yet been - specified for the current machine (MACHINE): - MACHINE = \"$MACHINE\"" - fi - ;; -# -esac +env_init_scripts_fps_str="( "$(printf "\"%s\" " "${ENV_INIT_SCRIPTS_FPS[@]}")")" +init_env env_init_scripts_fps="${env_init_scripts_fps_str}" # #----------------------------------------------------------------------- # @@ -131,7 +90,6 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # - machine=$(echo_lowercase $MACHINE) env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" module use "${SR_WX_APP_TOP_DIR}/env" diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index a707901dbe..14a36a0121 100755 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -1,14 +1,11 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk local external_file_fmt external_model location - external_model=${1} external_file_fmt=${2} @@ -35,8 +32,14 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/glade/u/apps/ch/opt/lmod/8.1.7/lmod/8.1.7/init/sh} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" @@ -71,6 +74,3 @@ MET_BIN_EXEC="bin" TEST_PREGEN_BASEDIR=/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen TEST_COMINgfs=/glade/scratch/ketefian/NCO_dirs/COMGFS TEST_EXTRN_MDL_SOURCE_BASEDIR=/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index 864a51c845..656d7d66e9 100755 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -26,7 +24,6 @@ function file_location() { echo ${location:-} } - EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ ${FV3GFS_FILE_FMT_ICS})} @@ -34,8 +31,14 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" @@ -75,7 +78,3 @@ TEST_COMINgfs=/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS TEST_EXTRN_MDL_SOURCE_BASEDIR=/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir - - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index 9b5472ecfd..9403fc2039 100755 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -41,7 +39,6 @@ function file_location() { echo ${location:-} } - EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ ${FV3GFS_FILE_FMT_ICS})} @@ -49,8 +46,14 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" @@ -80,7 +83,3 @@ RUN_CMD_POST="srun" TEST_PREGEN_BASEDIR=/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen TEST_COMINgfs=/lfs1/HFIP/hwrf-data/hafs-input/COMGFS TEST_EXTRN_MDL_SOURCE_BASEDIR=/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files - - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh index a9245f5216..10ffbf8027 100755 --- a/ush/machine/odin.sh +++ b/ush/machine/odin.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -36,6 +34,15 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=() +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + # Architecture information WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE=${NCORES_PER_NODE:-24} @@ -59,6 +66,3 @@ RUN_CMD_SERIAL="srun -n 1" RUN_CMD_UTILS='srun -n $nprocs' RUN_CMD_FCST='srun -n ${PE_MEMBER01}' RUN_CMD_POST="srun -n 1" - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index 9ed7223752..cfaee192b2 100755 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -25,7 +23,6 @@ function file_location() { } - EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ ${FV3GFS_FILE_FMT_ICS})} @@ -33,8 +30,14 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/apps/lmod/lmod/init/sh} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" @@ -62,6 +65,3 @@ RUN_CMD_POST="srun" # Test Data Locations TEST_EXTRN_MDL_SOURCE_BASEDIR=/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh index 0ba9436805..50c525d3fc 100755 --- a/ush/machine/stampede.sh +++ b/ush/machine/stampede.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -36,6 +34,15 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=() +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + # Architecture information WORKFLOW_MANAGER="rocoto" NCORES_PER_NODE="${NCORES_PER_NODE:-68}" @@ -59,6 +66,3 @@ RUN_CMD_SERIAL="time" RUN_CMD_UTILS='ibrun -np $nprocs' RUN_CMD_FCST='ibrun -np $nprocs' RUN_CMD_POST='ibrun -np $nprocs' - -ulimit -s unlimited -ulimit -a diff --git a/ush/machine/wcoss_cray.sh b/ush/machine/wcoss_cray.sh deleted file mode 100755 index 9d2836df5e..0000000000 --- a/ush/machine/wcoss_cray.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -set -x - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - case ${external_model} in - - "FV3GFS") - location='/gpfs/dell1/nco/ops/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' - ;; - "RAP") - location='/gpfs/hps/nco/ops/com/rap/prod' - ;; - "HRRR") - location='/gpfs/hps/nco/ops/com/hrrr/prod' - ;; - "NAM") - location='/gpfs/dell1/nco/ops/com/nam/prod' - ;; - "*") - print_err_msg_exit"\ - External model \'${external_model}\' does not have a default - location on Jet Please set a user-defined file location." - ;; - - esac - echo ${location:-} -} - - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} - -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/opt/modules/default/init/sh} - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-24} -SCHED=${SCHED:-"lsfcray"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} -QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} -QUEUE_FCST=${QUEUE_FCST:-"dev"} -RELATIVE_LINK_FLAG="" - -# UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} -FIXaer=${FIXaer:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} -FIXlut=${FIXlut:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen"} - -# MET Installation Locations -MET_INSTALL_DIR="/gpfs/hps3/emc/meso/noscrub/emc.metplus/met/10.0.0" -METPLUS_PATH="/gpfs/hps3/emc/meso/noscrub/emc.metplus/METplus/METplus-4.0.0" -CCPA_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" -MRMS_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" -NDAS_OBS_DIR="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" -MET_BIN_EXEC="exec" - -# Test Data Locations -TEST_PREGEN_BASEDIR=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/FV3LAM_pregen -TEST_COMINgfs=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/COMGFS -TEST_EXTRN_MDL_SOURCE_BASEDIR=/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 8ae99fdc1f..249f403d8f 100755 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x - function file_location() { # Return the default location of external model files on disk @@ -33,8 +31,8 @@ function file_location() { esac echo ${location:-} -} +} EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ @@ -43,8 +41,14 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} -# System Installations -MODULE_INIT_PATH=${MODULE_INIT_PATH:-/opt/modules/default/init/sh} +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" "/usrx/local/prod/lmod/lmod/init/sh" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' # Architecture information WORKFLOW_MANAGER="rocoto" @@ -62,6 +66,12 @@ TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fi SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen"} +# Commands to run +RUN_CMD_SERIAL="mpirun" +RUN_CMD_UTILS="mpirun" +RUN_CMD_FCST='mpirun -l -np ${PE_MEMBER01}' +RUN_CMD_POST="mpirun" + # MET Installation Locations MET_INSTALL_DIR="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" METPLUS_PATH="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" @@ -75,4 +85,3 @@ MET_BIN_EXEC="exec" TEST_PREGEN_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen TEST_COMINgfs=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS TEST_EXTRN_MDL_SOURCE_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files - diff --git a/ush/setup.sh b/ush/setup.sh index 407d20d5c2..5d41119b32 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -427,7 +427,7 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" # RELATIVE_LINK_FLAG="--relative" MACHINE_FILE=${MACHINE_FILE:-${USHDIR}/machine/$(echo_lowercase $MACHINE).sh} -source ${MACHINE_FILE} +source $USHDIR/source_machine_file.sh if [ -z "${NCORES_PER_NODE:-}" ]; then print_err_msg_exit "\ @@ -1342,7 +1342,7 @@ WFLOW_LAUNCH_SCRIPT_FP="$USHDIR/${WFLOW_LAUNCH_SCRIPT_FN}" WFLOW_LAUNCH_LOG_FP="$EXPTDIR/${WFLOW_LAUNCH_LOG_FN}" if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then CRONTAB_LINE="*/${CRON_RELAUNCH_INTVL_MNTS} * * * * cd $EXPTDIR && \ -./${WFLOW_LAUNCH_SCRIPT_FN} >> ./${WFLOW_LAUNCH_LOG_FN} 2>&1" +./${WFLOW_LAUNCH_SCRIPT_FN} called_from_cron=\"TRUE\" >> ./${WFLOW_LAUNCH_LOG_FN} 2>&1" else CRONTAB_LINE="" fi diff --git a/ush/source_machine_file.sh b/ush/source_machine_file.sh new file mode 100644 index 0000000000..532335962b --- /dev/null +++ b/ush/source_machine_file.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# +# This script is a wrapper that sources the machine file but also allows +# for certain pre- and post-source commands to be executed (e.g. setting +# of "set -x" shell option depending on the DEBUG variable). +# + +if [ -z ${DEBUG+x} ] || [ "$DEBUG" != "TRUE" ]; then + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +else + { save_shell_opts; set -u -x; } > /dev/null 2>&1 +fi + +source ${MACHINE_FILE} + +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 05b97e7e80..48e53d81a6 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -177,7 +177,7 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_GRID_TN;" "&JOBSDIR;/JREGIONAL_MAKE_GRID" - {%- if machine in ["WCOSS_DELL_P3", "WCOSS_CRAY"] %} + {%- if machine in ["WCOSS_DELL_P3"] %} {{ nnodes_make_grid }}:ppn=1 {%- else %} {{ nnodes_make_grid }}:ppn={{ ppn_make_grid }} @@ -200,7 +200,7 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_OROG_TN;" "&JOBSDIR;/JREGIONAL_MAKE_OROG" - {%- if machine in ["WCOSS_DELL_P3", "WCOSS_CRAY"] %} + {%- if machine in ["WCOSS_DELL_P3"] %} {{ nnodes_make_orog }}:ppn=1 {%- else %} {{ nnodes_make_orog }}:ppn={{ ppn_make_orog }} @@ -264,9 +264,6 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_ICS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" {%- if machine in ["WCOSS_DELL_P3"] %} 2048M-R affinity[core] @@ -294,9 +291,6 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_LBCS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" {%- if machine in ["WCOSS_DELL_P3"] %} 2048M-R affinity[core] @@ -698,9 +692,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_CCPA" {%- if machine in ["WCOSS_DELL_P3"] %} 2048M-R affinity[core] @@ -731,9 +722,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_MRMS" {%- if machine in ["WCOSS_DELL_P3"] %} 2048M-R affinity[core] @@ -765,9 +753,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; - {%- if machine in ["WCOSS_CRAY"] %} - - {%- endif %} &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_NDAS" {%- if machine in ["WCOSS_DELL_P3"] %} 2048M-R affinity[core] diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index e80721d51e..2e497c030a 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") +valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_WORKFLOW_MANAGER=("rocoto" "none") From f2c0c7b5b127b1f581fa28dbc9152884ed797c80 Mon Sep 17 00:00:00 2001 From: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Date: Wed, 2 Mar 2022 14:48:19 -0700 Subject: [PATCH 125/203] Fix to add missing "/" UPP namelist (#688) Co-authored-by: Tracy --- scripts/exregional_run_post.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index a2aea08d2c..c7aafd3997 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -238,6 +238,7 @@ grib='grib2' DateStr='${post_yyyy}-${post_mm}-${post_dd}_${post_hh}:${post_mn}:00' MODELNAME='FV3R' fileNameFlux='${phy_file}' +/ &NAMPGB KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,${post_itag_add} From 48eace2675238f886315217e830c93c81c4ddba3 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Fri, 4 Mar 2022 09:41:09 -0700 Subject: [PATCH 126/203] Fix bug for crontab feature when user's cron table is empty (#690) ## DESCRIPTION OF CHANGES: When a user's `cron` table is empty, the `get_crontab_contents` function does not set the return variable properly due to a bug in the way `printf -v ...` is called towards the end of that function. This PR fixes that bug as well as some others found after testing with an initially empty user cron table. ## TESTS CONDUCTED: On Hera, Jet, and Cheyenne, ran the WE2E test `deactivate_tasks`, starting with both an empty `cron` table and a non-empty one. All tests passed with the `cron` table being modified properly at the start and end of the test. ## CONTRIBUTORS (optional): @chan-hoo pointed out this error on Hera. --- ush/generate_FV3LAM_wflow.sh | 62 ++++++++++++++++++++++-------------- ush/get_crontab_contents.sh | 6 ++-- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index de466fb1be..39793b6863 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -506,33 +506,40 @@ Copying contents of user cron table to backup file: get_crontab_contents called_from_cron=${called_from_cron} \ outvarname_crontab_cmd="crontab_cmd" \ outvarname_crontab_contents="crontab_contents" - echo "${crontab_contents}" > "${crontab_backup_fp}" -# -# Below, we use "grep" to determine whether the crontab line that the -# variable CRONTAB_LINE contains is already present in the cron table. -# For that purpose, we need to escape the asterisks in the string in -# CRONTAB_LINE with backslashes. Do this next. -# + # To create the backup crontab file and add a new job to the user's + # existing cron table, use the "printf" command, not "echo", to print + # out variables. This is because "echo" will add a newline at the end + # of its output even if its input argument is a null string, resulting + # in extranous blank lines in the backup crontab file and/or the cron + # table itself. Using "printf" prevents the appearance of these blank + # lines. + printf "%s" "${crontab_contents}" > "${crontab_backup_fp}" + # + # Below, we use "grep" to determine whether the crontab line that the + # variable CRONTAB_LINE contains is already present in the cron table. + # For that purpose, we need to escape the asterisks in the string in + # CRONTAB_LINE with backslashes. Do this next. + # crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ $SED -r -e "s%[*]%\\\\*%g" ) -# -# In the grep command below, the "^" at the beginning of the string -# passed to grep is a start-of-line anchor, and the "$" at the end is -# an end-of-line anchor. Thus, in order for grep to find a match on -# any given line of the cron table's contents, that line must contain -# exactly the string in the variable crontab_line_esc_astr without any -# leading or trailing characters. This is to eliminate situations in -# which a line in the cron table contains the string in crontab_line_esc_astr -# but is precedeeded, for example, by the comment character "#" (in which -# case cron ignores that line) and/or is followed by further commands -# that are not part of the string in crontab_line_esc_astr (in which -# case it does something more than the command portion of the string in -# crontab_line_esc_astr does). -# + # + # In the grep command below, the "^" at the beginning of the string + # passed to grep is a start-of-line anchor, and the "$" at the end is + # an end-of-line anchor. Thus, in order for grep to find a match on + # any given line of the cron table's contents, that line must contain + # exactly the string in the variable crontab_line_esc_astr without any + # leading or trailing characters. This is to eliminate situations in + # which a line in the cron table contains the string in crontab_line_esc_astr + # but is precedeeded, for example, by the comment character "#" (in which + # case cron ignores that line) and/or is followed by further commands + # that are not part of the string in crontab_line_esc_astr (in which + # case it does something more than the command portion of the string in + # crontab_line_esc_astr does). + # if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then grep_output=$( grep "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" ) else - grep_output=$( echo "${crontab_contents}" | grep "^${crontab_line_esc_astr}$" ) + grep_output=$( printf "%s" "${crontab_contents}" | grep "^${crontab_line_esc_astr}$" ) fi exit_status=$? @@ -551,9 +558,16 @@ resubmit SRW workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - echo "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" + printf "%s" "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" else - ( echo "${crontab_contents}"; echo "${CRONTAB_LINE}" ) | ${crontab_cmd} + # Add a newline to the end of crontab_contents only if it is not empty. + # This is needed so that when CRONTAB_LINE is printed out, it appears on + # a separate line. + crontab_contents=${crontab_contents:+"${crontab_contents}"$'\n'} + # When printing CRONTAB_LINE, add a newline at the end. This is necessary + # on certain machines (e.g. Cheyenne) while on others, it doesn't make + # a difference. + ( printf "%s" "${crontab_contents}"; printf "%s\n" "${CRONTAB_LINE}" ) | ${crontab_cmd} fi fi diff --git a/ush/get_crontab_contents.sh b/ush/get_crontab_contents.sh index dbf683790b..9aae441156 100644 --- a/ush/get_crontab_contents.sh +++ b/ush/get_crontab_contents.sh @@ -70,13 +70,13 @@ function get_crontab_contents() { # copy of this header to the user's cron table. # if [ "$MACHINE" = "CHEYENNE" ]; then - __crontab_contents__=$( echo "${__crontab_contents__}" | tail -n +4 ) + __crontab_contents__=$( printf "%s" "${__crontab_contents__}" | tail -n +4 ) fi # # Set output variables. # - printf -v ${outvarname_crontab_cmd} "${__crontab_cmd__}" - printf -v ${outvarname_crontab_contents} "${__crontab_contents__}" + printf -v ${outvarname_crontab_cmd} "%s" "${__crontab_cmd__}" + printf -v ${outvarname_crontab_contents} "%s" "${__crontab_contents__}" { restore_shell_opts; } > /dev/null 2>&1 From 337df578596f47a15f06d957c9e9a52fd5e08881 Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Fri, 4 Mar 2022 15:30:45 -0600 Subject: [PATCH 127/203] Feature/singularity addition (#686) * Create singularity.sh * added machine/sing script for review we need to merge the change and delete this file later. * added singularity machine value * fixed mpirun command name and options: singularity * cleaned up singularity script * Update valid_param_vals.sh * Update singularity.sh * Update singularity.sh * Update singularity.sh Removed empty string MET and test variables. Co-authored-by: JONG KIM --- ush/machine/singularity.sh | 64 ++++++++++++++++++++++++++++++++++++++ ush/valid_param_vals.sh | 2 +- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 ush/machine/singularity.sh diff --git a/ush/machine/singularity.sh b/ush/machine/singularity.sh new file mode 100644 index 0000000000..36325d30b2 --- /dev/null +++ b/ush/machine/singularity.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "*") + print_info_msg"\ + External model \'${external_model}\' does not have a default + location. Please set a user-defined file location." + ;; + + esac + echo ${location:-} + +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/usr/share/lmod/6.6/init/profile" ) +fi + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-40} +SCHED=${SCHED:-"slurm"} +PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} +PARTITION_HPSS=${PARTITION_HPSS:-"service"} +QUEUE_HPSS=${QUEUE_HPSS:-"batch"} +PARTITION_FCST=${PARTITION_FCST:-""} +QUEUE_FCST=${QUEUE_FCST:-"batch"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/contrib/global/glopara/fix/fix_am"} +FIXaer=${FIXaer:-"/contrib/global/glopara/fix/fix_aer"} +FIXlut=${FIXlut:-"/contrib/global/glopara/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/contrib/global/glopara/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/global/glopara/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} + +RUN_CMD_SERIAL="time" +RUN_CMD_UTILS='mpirun -np $nprocs --oversubscribe' +RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} --oversubscribe' +RUN_CMD_POST='mpirun -np $nprocs --oversubscribe' diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 2e497c030a..4f5641a8ee 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") +valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "SINGULARITY") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_WORKFLOW_MANAGER=("rocoto" "none") From cc21ad74c1a08c34abaabdce79b8be1c283a4d5b Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:30:46 -0700 Subject: [PATCH 128/203] Bug fixes in machine files (#692) ## DESCRIPTION OF CHANGES: Cleaning up bugs in the machine files. The first bug prompted this PR, and the rest were found subsequently. The bugs (and their fixes) are as follows: 1) A space is missing after the `print_info_msg` and `print_err_msg_exit` function calls in the `file_location` functions. Inserting a space gets passed this bug, but subsequent issues were found as described below. **For machine files that call the `print_info_msg` function in `file_location` (`cheyenne.sh`, `hera.sh`, `jet.sh`, and `orion.sh`):** Fixing this bug leads to other failures because when the "*" stanza is encountered in the `file_location` function, the `EXTRN_MDL_SYSBASEDIR_ICS|LBCS` variable gets set to the message that `file_location` returns. Since that message contains spaces, it leads to other failures in downstream scripts (the ex-scripts). Simply removing the printing out of the message (thus causing `EXTRN_MDL_SYSBASEDIR_ICS|LBCS` to be set to a null string) fixes the failures, so this was the fix implemented. If desired, a message for an empty value for `EXTRN_MDL_SYSBASEDIR_ICS|LBCS` can be placed in another script (where those variables are used). **For machine files that use `print_err_msg_exit` in `file_location` (`stampede.sh` and `wcoss_dell_p3.sh`):** These should not exit if the file location is not available since the experiment can still complete successfully. So just removing the `print_err_msg_exit` call should work (and make the behavior of these machine files consistent with the set above). 2) In all the machine files, the variable `FV3GFS_FILE_FMT_ICS` should be changed to `FV3GFS_FILE_FMT_LBCS` in the definition of `EXTRN_MDL_SYSBASEDIR_LBCS`. This was fixed in all the files. 3) In `stampede.sh`, a variable named `SYSBASEDIR_ICS` is defined. This is a typo. Modify to `EXTRN_MDL_SYSBASEDIR_ICS`. ## TESTS CONDUCTED: Ran the WE2E test `grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR` on: * Hera -- successful * Jet -- successful except for UPP tasks * Cheyenne -- successful except for UPP tasks The UPP task failures are new and being experienced by other PRs as well (e.g. #689). The original issue with machine files seems resolved. ## CONTRIBUTORS (optional): @JeffBeck-NOAA encountered and reported the original error. --- ush/machine/cheyenne.sh | 15 ++++++--------- ush/machine/hera.sh | 20 +++++++++----------- ush/machine/jet.sh | 17 ++++++++--------- ush/machine/odin.sh | 10 ++++------ ush/machine/orion.sh | 15 ++++----------- ush/machine/singularity.sh | 16 ++++------------ ush/machine/stampede.sh | 13 +++++-------- ush/machine/wcoss_dell_p3.sh | 17 ++++++----------- 8 files changed, 46 insertions(+), 77 deletions(-) mode change 100755 => 100644 ush/machine/cheyenne.sh mode change 100755 => 100644 ush/machine/hera.sh mode change 100755 => 100644 ush/machine/jet.sh mode change 100755 => 100644 ush/machine/odin.sh mode change 100755 => 100644 ush/machine/orion.sh mode change 100755 => 100644 ush/machine/stampede.sh mode change 100755 => 100644 ush/machine/wcoss_dell_p3.sh diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh old mode 100755 new mode 100644 index 14a36a0121..c55d050f93 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -9,16 +9,12 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in "FV3GFS") location='/glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh}' ;; - "*") - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Cheyenne. Please set a user-defined file location." - ;; esac echo ${location:-} @@ -30,7 +26,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -57,6 +53,7 @@ TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS='mpirun -np $nprocs' RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' @@ -71,6 +68,6 @@ NDAS_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc" MET_BIN_EXEC="bin" # Test Data Locations -TEST_PREGEN_BASEDIR=/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen -TEST_COMINgfs=/glade/scratch/ketefian/NCO_dirs/COMGFS -TEST_EXTRN_MDL_SOURCE_BASEDIR=/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files +TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" +TEST_COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh old mode 100755 new mode 100644 index 656d7d66e9..6f57917053 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -9,19 +9,16 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in "FV3GFS") location='/scratch1/NCEPDEV/rstprod/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Hera. Will try to pull from HPSS" - ;; esac echo ${location:-} + } EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ @@ -29,7 +26,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -59,6 +56,7 @@ TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS="srun" RUN_CMD_FCST="srun" @@ -73,8 +71,8 @@ NDAS_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc" MET_BIN_EXEC="bin" # Test Data Locations -TEST_PREGEN_BASEDIR=/scratch2/BMC/det/FV3LAM_pregen -TEST_COMINgfs=/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS -TEST_EXTRN_MDL_SOURCE_BASEDIR=/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files -TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir -TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS=/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir +TEST_PREGEN_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" +TEST_COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" +TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" +TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh old mode 100755 new mode 100644 index 9403fc2039..800a0a6928 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -9,7 +9,9 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in + "FV3GFS") case $external_file_fmt in "nemsio") @@ -29,14 +31,10 @@ function file_location() { "HRRR") location='/public/data/grids/hrrr/conus/wrfprs/grib2' ;; - "*") - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Jet. Will try to pull from HPSS." - ;; esac echo ${location:-} + } EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ @@ -44,7 +42,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -74,12 +72,13 @@ TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS="srun" RUN_CMD_FCST="srun" RUN_CMD_POST="srun" # Test Data Locations -TEST_PREGEN_BASEDIR=/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen -TEST_COMINgfs=/lfs1/HFIP/hwrf-data/hafs-input/COMGFS -TEST_EXTRN_MDL_SOURCE_BASEDIR=/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files +TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" +TEST_COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh old mode 100755 new mode 100644 index 10ffbf8027..1bceaa873e --- a/ush/machine/odin.sh +++ b/ush/machine/odin.sh @@ -9,6 +9,7 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in "GSMGFS") @@ -17,11 +18,7 @@ function file_location() { "FV3GFS") location='/scratch/ywang/test_runs/FV3_regional/gfs/${yyyymmdd}' ;; - "*") - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Odin Please set a user-defined file location." - ;; + esac echo ${location:-} @@ -32,7 +29,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -62,6 +59,7 @@ TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +# Run commands for executables RUN_CMD_SERIAL="srun -n 1" RUN_CMD_UTILS='srun -n $nprocs' RUN_CMD_FCST='srun -n ${PE_MEMBER01}' diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh old mode 100755 new mode 100644 index cfaee192b2..5f4abc0d9c --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -10,25 +10,17 @@ function file_location() { external_model=${1} external_file_fmt=${2} - case ${external_model} in - - "*") - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Orion. Please set a user-defined file location." - ;; - - esac + location="" echo ${location:-} } EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -58,6 +50,7 @@ TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS="srun" RUN_CMD_FCST='srun -n ${PE_MEMBER01}' diff --git a/ush/machine/singularity.sh b/ush/machine/singularity.sh index 36325d30b2..528e1dbd5c 100644 --- a/ush/machine/singularity.sh +++ b/ush/machine/singularity.sh @@ -9,26 +9,17 @@ function file_location() { external_model=${1} external_file_fmt=${2} - case ${external_model} in - - "*") - print_info_msg"\ - External model \'${external_model}\' does not have a default - location. Please set a user-defined file location." - ;; - - esac + location="" echo ${location:-} } - EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -58,6 +49,7 @@ TOPO_DIR=${TOPO_DIR:-"/contrib/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/global/glopara/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS='mpirun -np $nprocs --oversubscribe' RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} --oversubscribe' diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh old mode 100755 new mode 100644 index 50c525d3fc..41afa5fc17 --- a/ush/machine/stampede.sh +++ b/ush/machine/stampede.sh @@ -9,6 +9,7 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in "GSMGFS") @@ -16,23 +17,18 @@ function file_location() { "FV3GFS") location='/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001' ;; - "*") - print_err_msg_exit"\ - External model \'${external_model}\' does not have a default - location on Jet Please set a user-defined file location." - ;; esac echo ${location:-} -} +} -SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -62,6 +58,7 @@ TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +# Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS='ibrun -np $nprocs' RUN_CMD_FCST='ibrun -np $nprocs' diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh old mode 100755 new mode 100644 index 249f403d8f..5ed70ddc9b --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -9,6 +9,7 @@ function file_location() { external_model=${1} external_file_fmt=${2} + location="" case ${external_model} in "FV3GFS") @@ -23,11 +24,6 @@ function file_location() { "NAM") location='/gpfs/dell1/nco/ops/com/nam/prod' ;; - "*") - print_err_msg_exit"\ - External model \'${external_model}\' does not have a default - location on Jet Please set a user-defined file location." - ;; esac echo ${location:-} @@ -39,7 +35,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${FV3GFS_FILE_FMT_ICS})} EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} + ${FV3GFS_FILE_FMT_LBCS})} # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). @@ -66,7 +62,7 @@ TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fi SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen"} -# Commands to run +# Run commands for executables RUN_CMD_SERIAL="mpirun" RUN_CMD_UTILS="mpirun" RUN_CMD_FCST='mpirun -l -np ${PE_MEMBER01}' @@ -80,8 +76,7 @@ MRMS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" NDAS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" MET_BIN_EXEC="exec" - # Test Data Locations -TEST_PREGEN_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen -TEST_COMINgfs=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS -TEST_EXTRN_MDL_SOURCE_BASEDIR=/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files +TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" +TEST_COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" From 5790920a25739bc313ef06824f85ff7dd2fe14fd Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Tue, 8 Mar 2022 13:29:27 -0500 Subject: [PATCH 129/203] Add NOAA cloud platforms to SRW (#689) * Tweaks for running with containers on azure * added config.sh for GST on azure * added AWS to load_modules_run_task.sh * working on bare metal now * Changing to azure, aws, and singularity * updates for singularity * tweaks for running using singularity exec * tweaks for running using singularity exec * Converting to a single noaacloud type * slight changes to config.sh for aws * update machine file * added missing slash to namelist * changes for intel * more cleanup * cleaned up commented lines --- ush/machine/noaacloud.sh | 71 ++++++++++++++++++++++++++++++++++++++++ ush/valid_param_vals.sh | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100755 ush/machine/noaacloud.sh diff --git a/ush/machine/noaacloud.sh b/ush/machine/noaacloud.sh new file mode 100755 index 0000000000..fe42766612 --- /dev/null +++ b/ush/machine/noaacloud.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/contrib/GST/model_data/FV3GFS/${yyyymmdd}${hh}' + ;; + *) + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Hera. Will try to pull from HPSS" + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="rocoto" +NCORES_PER_NODE=${NCORES_PER_NODE:-36} +SCHED=${SCHED:-"slurm"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/contrib/EPIC/fix/fix_am"} +FIXaer=${FIXaer:-"/contrib/EPIC/fix/fix_aer"} +FIXlut=${FIXlut:-"/contrib/EPIC/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/contrib/EPIC/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/EPIC/fix/fix_sfc_climo"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} + +RUN_CMD_SERIAL="time" +#Run Commands currently differ for GNU/openmpi +#RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' +#RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' +#RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' +RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' +RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' +RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' + +# MET Installation Locations +# MET Plus is not yet supported on noaacloud + diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 4f5641a8ee..a9bfa845ea 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "SINGULARITY") +valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_WORKFLOW_MANAGER=("rocoto" "none") From 8daa824b3e566fef5e6c87c49a84e8eff9b0ade9 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 9 Mar 2022 13:47:42 -0700 Subject: [PATCH 130/203] Add SPP to the SRW App (#685) * Add default namelist with SPP entries. * Changes necessary to run with SPP * Typo fix in generate script. * Changes to implement SPP. * Add comment regarding use of SPP. * Pass N_VAR_SPP to the var_defns.sh file. * Add spp_wts_pbl to the FV3_GSD_SAR SDF diag_table file * Remove the contents of the &nam_spperts stanza and n_var_spp from the namelist when not using SPP. * Remove SPP namelist entries from the template input.nml file. These values are now handled in the generate script if using SPP. * Add MYNN SFC pattern variable to the diag_table file for the FV3_GSD_SAR SDF * Add 'sfc' perturbation option to SPP * Add iseed array to namelist generation * Changes to add "rad" and "gwd" SPP perturbations. * Add SPP and ad-hoc stochastic physics to SDFs. * Add LSM SPP functionality to the SRW App. * Add random number generation for LSM SPP iseeds in ensemble mode. * Fix undeclared variable for LSM SPP. * Add if statement for LSM SPP namelist entries and set fhcyc to 999 if LSM SPP is turned on. * Modify how namelist settings are applied for LSM SPP. * Fix implementation of fhcyc changes. * Typo fix. * Add do_gsl* namelist entries to YAML file for FV3_HRRR SDF. * Remove diss_est from the diag table files since it's unavailable for now. * Change LSM SPP perturbation seed to be the same as all other SPP. * Changes to FV3_HRRR field_table * Change to the FV3_HRRR field_table file * Update name for SPP block in input.nml * Change SPP_LSM_* to LSM_SPP_* * Remove space. * Update descriptions of LSM perturbations. * Shut off PET file generation. * Modifications for land and SPP perturbations (templates, namelists, default values, etc.) * Remove diss_est from diag_table files and remove the FV3_GSD_SAR and FV3_GSD_v0 diag_table files. * Fix in setup.sh for LSM SPP * Update Thompson MP SPP settings. * Add back FV3_GSD_SAR and FV3_GSD_v0 SDFs. Will be removed in future, separate PR. * Requested modifications to the input.nml template and in-line documentation changes. * Fix check for LSM SPP namelist settings * Fix to LSM SPP namelist check. * Changes request based on PR review. * Changes requested in PR review. * Variable descriptions. * Fix comment formatting. * Fix MET/METplus/obs paths in machine files to allow for user-defined settings in config.sh * Only add/modify stochastic physics namelist entries when running with SPP, LSM SPP, SPPT, SHUM, or SKEB. --- ush/config_defaults.sh | 134 ++++++++++++------- ush/generate_FV3LAM_wflow.sh | 43 +++++- ush/machine/cheyenne.sh | 14 +- ush/machine/hera.sh | 14 +- ush/machine/wcoss_dell_p3.sh | 14 +- ush/set_FV3nml_stoch_params.sh | 49 ++++++- ush/setup.sh | 87 +++++++++++- ush/templates/diag_table.FV3_CPT_v0 | 1 - ush/templates/diag_table.FV3_GFS_2017_gfdlmp | 1 - ush/templates/diag_table.FV3_GFS_v15p2 | 1 - ush/templates/diag_table.FV3_GFS_v16 | 1 - ush/templates/diag_table.FV3_HRRR | 6 +- ush/templates/diag_table.FV3_RRFS_v1alpha | 8 ++ ush/templates/diag_table.FV3_RRFS_v1beta | 9 ++ ush/templates/field_table.FV3_HRRR | 2 +- ush/templates/input.nml.FV3 | 35 +---- ush/templates/nems.configure | 1 + ush/valid_param_vals.sh | 1 + 18 files changed, 304 insertions(+), 117 deletions(-) mode change 100644 => 100755 ush/templates/diag_table.FV3_CPT_v0 mode change 100644 => 100755 ush/templates/diag_table.FV3_GFS_2017_gfdlmp mode change 100644 => 100755 ush/templates/diag_table.FV3_GFS_v15p2 mode change 100644 => 100755 ush/templates/diag_table.FV3_GFS_v16 mode change 100644 => 100755 ush/templates/diag_table.FV3_HRRR mode change 100644 => 100755 ush/templates/diag_table.FV3_RRFS_v1alpha mode change 100644 => 100755 ush/templates/diag_table.FV3_RRFS_v1beta diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 1e59038ee5..4c7b4d7e8c 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -507,54 +507,54 @@ WRITE_DOPOST="FALSE" # # CCPA_OBS_DIR: # User-specified location of top-level directory where CCPA hourly -# precipitation files used by METplus are located. This parameter needs -# to be set for both user-provided observations and for observations -# that are retrieved from the NOAA HPSS (if the user has access) via -# the get_obs_ccpa_tn task (activated in workflow by setting  +# precipitation files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via +# the get_obs_ccpa_tn task (activated in workflow by setting # RUN_TASK_GET_OBS_CCPA="TRUE"). In the case of pulling observations # directly from NOAA HPSS, the data retrieved will be placed in this # directory. Please note, this path must be defind as # /full-path-to-obs/ccpa/proc. METplus is configured to verify 01-, -# 03-, 06-, and 24-h accumulated precipitation using hourly CCPA files.  +# 03-, 06-, and 24-h accumulated precipitation using hourly CCPA files. # METplus configuration files require the use of predetermined directory # structure and file names. Therefore, if the CCPA files are user # provided, they need to follow the anticipated naming structure: -# {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, where YYYY is the 4-digit -# valid year, MM the 2-digit valid month, DD the 2-digit valid day of -# the month, and HH the 2-digit valid hour of the day. In addition, a +# {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, where YYYY is the 4-digit +# valid year, MM the 2-digit valid month, DD the 2-digit valid day of +# the month, and HH the 2-digit valid hour of the day. In addition, a # caveat is noted for using hourly CCPA data. There is a problem with # the valid time in the metadata for files valid from 19 - 00 UTC (or -# files  under the '00' directory). The script to pull the CCPA data +# files under the '00' directory). The script to pull the CCPA data # from the NOAA HPSS has an example of how to account for this as well -# as organizing the data into a more intuitive format: +# as organizing the data into a more intuitive format: # regional_workflow/scripts/exregional_get_ccpa_files.sh. When a fix # is provided, it will be accounted for in the # exregional_get_ccpa_files.sh script. # # MRMS_OBS_DIR: -# User-specified location of top-level directory where MRMS composite -# reflectivity files used by METplus are located. This parameter needs -# to be set for both user-provided observations and for observations -# that are retrieved from the NOAA HPSS (if the user has access) via the -# get_obs_mrms_tn task (activated in workflow by setting  -# RUN_TASK_GET_OBS_MRMS="TRUE"). In the case of pulling observations +# User-specified location of top-level directory where MRMS composite +# reflectivity files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via the +# get_obs_mrms_tn task (activated in workflow by setting +# RUN_TASK_GET_OBS_MRMS="TRUE"). In the case of pulling observations # directly from NOAA HPSS, the data retrieved will be placed in this # directory. Please note, this path must be defind as # /full-path-to-obs/mrms/proc. METplus configuration files require the -# use of predetermined directory structure and file names. Therefore, if -# the MRMS files are user provided, they need to follow the anticipated +# use of predetermined directory structure and file names. Therefore, if +# the MRMS files are user provided, they need to follow the anticipated # naming structure: -# {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, -# where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD -# the 2-digit valid day of the month, HH the 2-digit valid hour of the +# {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, +# where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD +# the 2-digit valid day of the month, HH the 2-digit valid hour of the # day, mm the 2-digit valid minutes of the hour, and SS is the two-digit # valid seconds of the hour. In addition, METplus is configured to look # for a MRMS composite reflectivity file for the valid time of the # forecast being verified; since MRMS composite reflectivity files do # not always exactly match the valid time, a script, within the main # script to retrieve MRMS data from the NOAA HPSS, is used to identify -# and rename the MRMS composite reflectivity file to match the valid -# time of the forecast. The script to pull the MRMS data from the NOAA +# and rename the MRMS composite reflectivity file to match the valid +# time of the forecast. The script to pull the MRMS data from the NOAA # HPSS has an example of the expected file naming structure: # regional_workflow/scripts/exregional_get_mrms_files.sh. This script # calls the script used to identify the MRMS file closest to the valid @@ -562,23 +562,23 @@ WRITE_DOPOST="FALSE" # # NDAS_OBS_DIR: # User-specified location of top-level directory where NDAS prepbufr -# files used by METplus are located. This parameter needs to be set for -# both user-provided observations and for observations that are -# retrieved from the NOAA HPSS (if the user has access) via the +# files used by METplus are located. This parameter needs to be set for +# both user-provided observations and for observations that are +# retrieved from the NOAA HPSS (if the user has access) via the # get_obs_ndas_tn task (activated in workflow by setting  # RUN_TASK_GET_OBS_NDAS="TRUE"). In the case of pulling observations # directly from NOAA HPSS, the data retrieved will be placed in this # directory. Please note, this path must be defind as # /full-path-to-obs/ndas/proc. METplus is configured to verify # near-surface variables hourly and upper-air variables at times valid -# at 00 and 12 UTC with NDAS prepbufr files. METplus configuration files -# require the use of predetermined file names. Therefore, if the NDAS -# files are user provided, they need to follow the anticipated naming -# structure: prepbufr.ndas.{YYYYMMDDHH}, where YYYY is the 4-digit valid -# year, MM the 2-digit valid month, DD the 2-digit valid day of the -# month, and HH the 2-digit valid hour of the day. The script to pull +# at 00 and 12 UTC with NDAS prepbufr files. METplus configuration files +# require the use of predetermined file names. Therefore, if the NDAS +# files are user provided, they need to follow the anticipated naming +# structure: prepbufr.ndas.{YYYYMMDDHH}, where YYYY is the 4-digit valid +# year, MM the 2-digit valid month, DD the 2-digit valid day of the +# month, and HH the 2-digit valid hour of the day. The script to pull # the NDAS data from the NOAA HPSS has an example of how to rename the -# NDAS data into a more intuitive format with the valid time listed in +# NDAS data into a more intuitive format with the valid time listed in # the file name: regional_workflow/scripts/exregional_get_ndas_files.sh # #----------------------------------------------------------------------- @@ -1505,6 +1505,7 @@ FIXgsm_FILES_TO_COPY_TO_FIXam=( \ "global_zorclim.1x1.grb" \ "global_sfc_emissivity_idx.txt" \ "global_solarconstant_noaa_an.txt" \ +"global_albedo4.1x1.grb" \ "geo_em.d01.lat-lon.2.5m.HGT_M.nc" \ "HGT.Beljaars_filtered.lat-lon.30s_res.nc" \ "replace_with_FIXgsm_ozone_prodloss_filename" \ @@ -1555,6 +1556,7 @@ CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING=( \ "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt" \ "co2monthlycyc.txt | co2monthlycyc.txt" \ "global_h2oprdlos.f77 | global_h2o_pltc.f77" \ +"global_albedo4.1x1.grb | global_albedo4.1x1.grb" \ "global_zorclim.1x1.grb | global_zorclim.1x1.grb" \ "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt" \ "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt" \ @@ -1808,27 +1810,62 @@ USE_ZMTNBLCK="FALSE" # #----------------------------------------------------------------------- # -# Set default SPP stochastic physics options. -# Each SPP option is an array, applicable (in order) to the scheme/parameter -# listed in SPP_VAR_LIST. Enter each value of the array in config.sh as -# shown below without commas or single quotes (e.g., SPP_VAR_LIST= -# ( "pbl" "lsm" "mp" ). Both commas and single quotes will be added by -# Jinja when creating the namelist. +# Set default SPP stochastic physics options. Each SPP option is an array, +# applicable (in order) to the scheme/parameter listed in SPP_VAR_LIST. +# Enter each value of the array in config.sh as shown below without commas +# or single quotes (e.g., SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ). +# Both commas and single quotes will be added by Jinja when creating the +# namelist. # # Note that SPP is currently only available for specific physics schemes # used in the RAP/HRRR physics suite. Users need to be aware of which SDF # is chosen when turning this option on. # +# Patterns evolve and are applied at each time step. +# #----------------------------------------------------------------------- # -DO_SPP="FALSE" -SPP_VAR_LIST=( "pbl" ) -SPP_MAG_LIST=( "0.2" ) #Variable "spp_prt_list" in input.nml -SPP_LSCALE=( "150000.0" ) -SPP_TSCALE=( "21600.0" ) #Variable "spp_tau" in input.nml -SPP_SIGTOP1=( "0.1" ) -SPP_SIGTOP2=( "0.025" ) -SPP_STDDEV_CUTOFF=( "1.5" ) +DO_SPP="false" +SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ) +SPP_MAG_LIST=( "0.2" "0.2" "0.75" "0.2" "0.2" ) #Variable "spp_prt_list" in input.nml +SPP_LSCALE=( "150000.0" "150000.0" "150000.0" "150000.0" "150000.0" ) +SPP_TSCALE=( "21600.0" "21600.0" "21600.0" "21600.0" "21600.0" ) #Variable "spp_tau" in input.nml +SPP_SIGTOP1=( "0.1" "0.1" "0.1" "0.1" "0.1") +SPP_SIGTOP2=( "0.025" "0.025" "0.025" "0.025" "0.025" ) +SPP_STDDEV_CUTOFF=( "1.5" "1.5" "2.5" "1.5" "1.5" ) +ISEED_SPP=( "4" "4" "4" "4" "4" ) +# +#----------------------------------------------------------------------- +# +# Turn on SPP in Noah or RUC LSM (support for Noah MP is in progress). +# Please be aware of the SDF that you choose if you wish to turn on LSM +# SPP. +# +# SPP in LSM schemes is handled in the &nam_sfcperts namelist block +# instead of in &nam_sppperts, where all other SPP is implemented. +# +# The default perturbation frequency is determined by the fhcyc namelist +# entry. Since that parameter is set to zero in the SRW App, use +# LSM_SPP_EACH_STEP to perturb every time step. +# +# Perturbations to soil moisture content (SMC) are only applied at the +# first time step. +# +# LSM perturbations include SMC - soil moisture content (volume fraction), +# VGF - vegetation fraction, ALB - albedo, SAL - salinity, +# EMI - emissivity, ZOL - surface roughness (cm), and STC - soil temperature. +# +# Only five perturbations at a time can be applied currently, but all seven +# are shown below. In addition, only one unique iseed value is allowed +# at the moment, and is used for each pattern. +# +DO_LSM_SPP="false" #If true, sets lndp_type=2 +LSM_SPP_TSCALE=( "21600" "21600" "21600" "21600" "21600" "21600" "21600" ) +LSM_SPP_LSCALE=( "150000" "150000" "150000" "150000" "150000" "150000" "150000" ) +ISEED_LSM_SPP=( "9" ) +LSM_SPP_VAR_LIST=( "smc" "vgf" "alb" "sal" "emi" "zol" "stc" ) +LSM_SPP_MAG_LIST=( "0.2" "0.001" "0.001" "0.001" "0.001" "0.001" "0.2" ) +LSM_SPP_EACH_STEP="true" #Sets lndp_each_step=.true. # #----------------------------------------------------------------------- # @@ -1936,3 +1973,6 @@ OMP_STACKSIZE_RUN_FCST="1024m" KMP_AFFINITY_RUN_POST="scatter" OMP_NUM_THREADS_RUN_POST="1" OMP_STACKSIZE_RUN_POST="1024m" +# +#----------------------------------------------------------------------- +# diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 39793b6863..21db7b20a4 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -784,6 +784,12 @@ settings="\ 'do_shum': ${DO_SHUM}, 'do_sppt': ${DO_SPPT}, 'do_skeb': ${DO_SKEB}, + 'do_spp': ${DO_SPP}, + 'n_var_spp': ${N_VAR_SPP}, + 'n_var_lndp': ${N_VAR_LNDP}, + 'lndp_type': ${LNDP_TYPE}, + 'lndp_each_step': ${LSM_SPP_EACH_STEP}, + 'fhcyc': ${FHCYC_LSM_SPP_OR_NOT}, } 'nam_stochy': { 'shum': ${SHUM_MAG}, @@ -855,7 +861,42 @@ done # settings="$settings }" - +# +# Add the relevant SPP namelist variables to "settings" when running with +# SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. +# +settings="$settings +'nam_sppperts': {" +if [ "${DO_SPP}" = "TRUE" ]; then + settings="$settings + 'iseed_spp': [ $( printf "%s, " "${ISEED_SPP[@]}" ) ], + 'spp_lscale': [ $( printf "%s, " "${SPP_LSCALE[@]}" ) ], + 'spp_prt_list': [ $( printf "%s, " "${SPP_MAG_LIST[@]}" ) ], + 'spp_sigtop1': [ $( printf "%s, " "${SPP_SIGTOP1[@]}" ) ], + 'spp_sigtop2': [ $( printf "%s, " "${SPP_SIGTOP2[@]}" ) ], + 'spp_stddev_cutoff': [ $( printf "%s, " "${SPP_STDDEV_CUTOFF[@]}" ) ], + 'spp_tau': [ $( printf "%s, " "${SPP_TSCALE[@]}" ) ], + 'spp_var_list': [ $( printf "%s, " "${SPP_VAR_LIST[@]}" ) ]," +fi +settings="$settings + }" +# +# Add the relevant LSM SPP namelist variables to "settings" when running with +# LSM SPP turned on. +# +settings="$settings +'nam_sfcperts': {" +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + settings="$settings + 'lndp_type': ${LNDP_TYPE}, + 'lndp_tau': [ $( printf "%s, " "${LSM_SPP_TSCALE[@]}" ) ], + 'lndp_lscale': [ $( printf "%s, " "${LSM_SPP_LSCALE[@]}" ) ], + 'iseed_lndp': [ $( printf "%s, " "${ISEED_LSM_SPP[@]}" ) ], + 'lndp_var_list': [ $( printf "%s, " "${LSM_SPP_VAR_LIST[@]}" ) ], + 'lndp_prt_list': [ $( printf "%s, " "${LSM_SPP_MAG_LIST[@]}" ) ]," +fi +settings="$settings + }" print_info_msg $VERBOSE " The variable \"settings\" specifying values of the weather model's namelist variables has been set as follows: diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index c55d050f93..599e463528 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -59,13 +59,13 @@ RUN_CMD_UTILS='mpirun -np $nprocs' RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' RUN_CMD_POST='mpirun -np $nprocs' -# MET Installation Locations -MET_INSTALL_DIR="/glade/p/ral/jntp/MET/MET_releases/10.0.0" -METPLUS_PATH="/glade/p/ral/jntp/MET/METplus/METplus-4.0.0" -CCPA_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc" -MRMS_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc" -NDAS_OBS_DIR="/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc" -MET_BIN_EXEC="bin" +# MET/METplus-Related Paths +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.0.0"} +METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.0.0"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index 6f57917053..bbe65dd3db 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -62,13 +62,13 @@ RUN_CMD_UTILS="srun" RUN_CMD_FCST="srun" RUN_CMD_POST="srun" -# MET Installation Locations -MET_INSTALL_DIR="/contrib/met/10.0.0" -METPLUS_PATH="/contrib/METplus/METplus-4.0.0" -CCPA_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc" -MRMS_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc" -NDAS_OBS_DIR="/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc" -MET_BIN_EXEC="bin" +# MET/METplus-Related Paths +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.0.0"} +METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.0.0"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations TEST_PREGEN_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 5ed70ddc9b..03656438fb 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -68,13 +68,13 @@ RUN_CMD_UTILS="mpirun" RUN_CMD_FCST='mpirun -l -np ${PE_MEMBER01}' RUN_CMD_POST="mpirun" -# MET Installation Locations -MET_INSTALL_DIR="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" -METPLUS_PATH="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" -CCPA_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc" -MRMS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" -NDAS_OBS_DIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" -MET_BIN_EXEC="exec" +# MET/METplus-Related Paths +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0"} +METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} # Test Data Locations TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index 8f08bb566a..8abfc45d47 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -80,6 +80,9 @@ function set_FV3nml_stoch_params() { iseed_skeb \ iseed_sppt \ iseed_spp \ + iseed_lsm_spp \ + num_iseed_spp \ + num_iseed_lsm_spp \ settings # #----------------------------------------------------------------------- @@ -98,17 +101,45 @@ function set_FV3nml_stoch_params() { iseed_shum=$(( cdate*1000 + ensmem_num*10 + 2 )) iseed_skeb=$(( cdate*1000 + ensmem_num*10 + 3 )) iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) - iseed_spp=$(( cdate*1000 + ensmem_num*10 + 4 )) + iseed_lsm_spp=$(( cdate*1000 + ensmem_num*10 + 9)) - settings="\ + num_iseed_spp=${#ISEED_SPP[@]} + for (( i=0; i<${num_iseed_spp}; i++ )); do + iseed_spp[$i]=$(( cdate*1000 + ensmem_num*10 + ${ISEED_SPP[$i]} )) + done + + settings="" + + if [ ${DO_SPPT} = TRUE ] || [ ${DO_SHUM} = TRUE ] || [ ${DO_SKEB} = TRUE ]; then + + settings=$settings"\ 'nam_stochy': { 'iseed_shum': ${iseed_shum}, 'iseed_skeb': ${iseed_skeb}, 'iseed_sppt': ${iseed_sppt}, } -'nam_spperts': { - 'iseed_spp': ${iseed_spp}, - }" +" + fi + + if [ ${DO_SPP} = TRUE ]; then + + settings=$settings"\ +'nam_sppperts': { + 'iseed_spp': [ $( printf "%s, " "${iseed_spp[@]}" ) ] + } +" + fi + + if [ ${DO_LSM_SPP} = TRUE ]; then + + settings=$settings"\ +'nam_sfcperts': { + 'iseed_lndp': [ $( printf "%s, " "${iseed_lsm_spp[@]}" ) ] + } +" + fi + + if [ -n "$settings" ]; then $USHDIR/set_namelist.py -q \ -n ${FV3_NML_FP} \ @@ -125,6 +156,14 @@ failed. Parameters passed to this script are: Namelist settings specified on command line (these have highest precedence): settings = $settings" + + else + + print_info_msg "\ +The variable \"settings\" is empty, so not setting any namelist values." + + fi + # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index 5d41119b32..01470963d5 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -204,6 +204,9 @@ DO_SKEB=$(boolify $DO_SKEB) check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" DO_SPP=$(boolify $DO_SPP) + +check_var_valid_value "DO_LSM_SPP" "valid_vals_DO_LSM_SPP" +DO_LSM_SPP=$(boolify $DO_LSM_SPP) # #----------------------------------------------------------------------- # @@ -227,8 +230,9 @@ fi # #----------------------------------------------------------------------- # -# If running with SPP, count the number of entries in SPP_VAR_LIST to -# correctly set N_VAR_SPP, otherwise set it to zero. +# If running with SPP in MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or +# RRTMG, count the number of entries in SPP_VAR_LIST to correctly set +# N_VAR_SPP, otherwise set it to zero. # #----------------------------------------------------------------------- # @@ -239,6 +243,73 @@ fi # #----------------------------------------------------------------------- # +# If running with Noah or RUC-LSM SPP, count the number of entries in +# LSM_SPP_VAR_LIST to correctly set N_VAR_LNDP, otherwise set it to zero. +# Also set LNDP_TYPE to 2 for LSM SPP, otherwise set it to zero. Finally, +# initialize an "FHCYC_LSM_SPP" variable to 0 and set it to 999 if LSM SPP +# is turned on. This requirement is necessary since LSM SPP cannot run with +# FHCYC=0 at the moment, but FHCYC cannot be set to anything less than the +# length of the forecast either. A bug fix will be submitted to +# ufs-weather-model soon, at which point, this requirement can be removed +# from regional_workflow. +# +#----------------------------------------------------------------------- +# +N_VAR_LNDP=0 +LNDP_TYPE=0 +FHCYC_LSM_SPP_OR_NOT=0 +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + N_VAR_LNDP=${#LSM_SPP_VAR_LIST[@]} + LNDP_TYPE=2 + FHCYC_LSM_SPP_OR_NOT=999 +fi +# +#----------------------------------------------------------------------- +# +# If running with SPP, confirm that each SPP-related namelist value +# contains the same number of entries as N_VAR_SPP (set above to be equal +# to the number of entries in SPP_VAR_LIST). +# +#----------------------------------------------------------------------- +# +if [ "${DO_SPP}" = "TRUE" ]; then + if [ "${#SPP_MAG_LIST[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_LSCALE[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_TSCALE[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_SIGTOP1[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_SIGTOP2[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_STDDEV_CUTOFF[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#ISEED_SPP[@]}" != "${N_VAR_SPP}" ]; then + print_err_msg_exit "\ +All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist +variables set in ${CONFIG_FN} must be equal in number of entries to what is +found in SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"${#SPP_VAR_LIST[@]}\"" + fi +fi +# +#----------------------------------------------------------------------- +# +# If running with LSM SPP, confirm that each LSM SPP-related namelist +# value contains the same number of entries as N_VAR_LNDP (set above to +# be equal to the number of entries in LSM_SPP_VAR_LIST). +# +#----------------------------------------------------------------------- +# +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + if [ "${#LSM_SPP_MAG_LIST[@]}" != "${N_VAR_LNDP}" ] || \ + [ "${#LSM_SPP_LSCALE[@]}" != "${N_VAR_LNDP}" ] || \ + [ "${#LSM_SPP_TSCALE[@]}" != "${N_VAR_LNDP}" ]; then + print_err_msg_exit "\ +All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) +set in ${CONFIG_FN} must be equal in number of entries to what is found in +SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"${#LSM_SPP_VAR_LIST[@]}\"" + fi +fi +# +#----------------------------------------------------------------------- +# check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" SUB_HOURLY_POST=$(boolify $SUB_HOURLY_POST) # @@ -2606,13 +2677,19 @@ PE_MEMBER01='${PE_MEMBER01}' # #----------------------------------------------------------------------- # -# IF DO_SPP is set to \"TRUE\", N_VAR_SPP specifies the number of physics -# parameterizations that are perturbed with SPP. Otherwise, N_VAR_SPP -# is set 0. +# IF DO_SPP is set to "TRUE", N_VAR_SPP specifies the number of physics +# parameterizations that are perturbed with SPP. If DO_LSM_SPP is set to +# "TRUE", N_VAR_LNDP specifies the number of LSM parameters that are +# perturbed. LNDP_TYPE determines the way LSM perturbations are employed +# and FHCYC_LSM_SPP_OR_NOT sets FHCYC based on whether LSM perturbations +# are turned on or not. # #----------------------------------------------------------------------- # N_VAR_SPP='${N_VAR_SPP}' +N_VAR_LNDP='${N_VAR_LNDP}' +LNDP_TYPE='${LNDP_TYPE}' +FHCYC_LSM_SPP_OR_NOT='${FHCYC_LSM_SPP_OR_NOT}' " # # Done with constructing the contents of the variable definitions file, diff --git a/ush/templates/diag_table.FV3_CPT_v0 b/ush/templates/diag_table.FV3_CPT_v0 old mode 100644 new mode 100755 index 1145d17a3b..c3e906af4a --- a/ush/templates/diag_table.FV3_CPT_v0 +++ b/ush/templates/diag_table.FV3_CPT_v0 @@ -275,7 +275,6 @@ "gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 "gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 #============================================================================================= # diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp old mode 100644 new mode 100755 index f24f648d99..1b46eeb99a --- a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp +++ b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp @@ -274,7 +274,6 @@ "gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 "gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 #============================================================================================= # diff --git a/ush/templates/diag_table.FV3_GFS_v15p2 b/ush/templates/diag_table.FV3_GFS_v15p2 old mode 100644 new mode 100755 index cde3d0f759..d4b1af5320 --- a/ush/templates/diag_table.FV3_GFS_v15p2 +++ b/ush/templates/diag_table.FV3_GFS_v15p2 @@ -274,7 +274,6 @@ "gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 "gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 #============================================================================================= diff --git a/ush/templates/diag_table.FV3_GFS_v16 b/ush/templates/diag_table.FV3_GFS_v16 old mode 100644 new mode 100755 index cde3d0f759..d4b1af5320 --- a/ush/templates/diag_table.FV3_GFS_v16 +++ b/ush/templates/diag_table.FV3_GFS_v16 @@ -274,7 +274,6 @@ "gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 "gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 #============================================================================================= diff --git a/ush/templates/diag_table.FV3_HRRR b/ush/templates/diag_table.FV3_HRRR old mode 100644 new mode 100755 index 0d2849c346..fa16a7e107 --- a/ush/templates/diag_table.FV3_HRRR +++ b/ush/templates/diag_table.FV3_HRRR @@ -324,8 +324,12 @@ "gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 "gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 "gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_pbl", "spp_wts_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_sfc", "spp_wts_sfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_mp", "spp_wts_mp", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_gwd", "spp_wts_gwd", "fv3_history", "all", .false., "none", 2 #============================================================================================= # diff --git a/ush/templates/diag_table.FV3_RRFS_v1alpha b/ush/templates/diag_table.FV3_RRFS_v1alpha old mode 100644 new mode 100755 index 99b7652910..0bc7e358d4 --- a/ush/templates/diag_table.FV3_RRFS_v1alpha +++ b/ush/templates/diag_table.FV3_RRFS_v1alpha @@ -305,6 +305,14 @@ # Prognostic/diagnostic variables from RUC LSM "gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 "gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +# Stochastic physics +"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_pbl", "spp_wts_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_mp", "spp_wts_mp", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 #============================================================================================= # diff --git a/ush/templates/diag_table.FV3_RRFS_v1beta b/ush/templates/diag_table.FV3_RRFS_v1beta old mode 100644 new mode 100755 index 99b7652910..03ce373f4c --- a/ush/templates/diag_table.FV3_RRFS_v1beta +++ b/ush/templates/diag_table.FV3_RRFS_v1beta @@ -305,6 +305,15 @@ # Prognostic/diagnostic variables from RUC LSM "gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 "gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 +# Stochastic physics +"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_pbl", "spp_wts_pbl", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_sfc", "spp_wts_sfc", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_mp", "spp_wts_mp", "fv3_history", "all", .false., "none", 2 +"gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 #============================================================================================= # diff --git a/ush/templates/field_table.FV3_HRRR b/ush/templates/field_table.FV3_HRRR index fe96567e5f..fbfb0c3399 100644 --- a/ush/templates/field_table.FV3_HRRR +++ b/ush/templates/field_table.FV3_HRRR @@ -33,7 +33,7 @@ "TRACER", "atmos_mod", "water_nc" "longname", "cloud liquid water number concentration" "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "profile_type", "fixed", "surface_value=0.0" / # prognostic cloud ice number concentration "TRACER", "atmos_mod", "ice_nc" "longname", "cloud ice water number concentration" diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 450b1361a9..6327120005 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -141,9 +141,6 @@ debug = .false. do_mynnedmf = .true. do_mynnsfclay = .false. - do_shum = .false. - do_skeb = .false. - do_sppt = .false. dspheat = .true. effr_in = .true. fhcyc = 0.0 @@ -170,13 +167,11 @@ ivegsrc = 1 ldiag3d = .false. lheatstrg = .false. - lndp_type = 0 lradar = .true. lsm = 3 lsoil_lsm = 9 ltaerosol = .true. lwhtr = .true. - n_var_lndp = 0 nsradar_reset = 3600 nst_anl = .true. nstf_name = 2,1,0,0,0 @@ -201,36 +196,12 @@ / &nam_sfcperts - lndp_type = 0, - lndp_tau=21600, - lndp_lscale=500000, - iseed_lndp=2010, - lndp_var_list = 'smc', 'vgf', - lndp_prt_list = 0.2,0.1, +/ + +&nam_sppperts / &nam_stochy - iseed_shum = 2 - iseed_skeb = 3 - iseed_sppt = 1 - new_lscale = .true. - shum = 0.006 - shum_lscale = 150.E3 - shum_tau = 21600 - shumint = 3600 - skeb = 0.5 - skeb_lscale = 150.E3 - skeb_tau = 21600.0 - skeb_vdof = 10 - skebint = 3600 - skebnorm = 1 - sppt = 1.0 - sppt_logit = .true. - sppt_lscale = 150.E3 - sppt_sfclimit = .true. - sppt_tau = 2.16E4 - spptint = 3600 - use_zmtnblck = .false. / &namsfc diff --git a/ush/templates/nems.configure b/ush/templates/nems.configure index b16668ad80..86b095464b 100644 --- a/ush/templates/nems.configure +++ b/ush/templates/nems.configure @@ -3,3 +3,4 @@ EARTH_component_list: ATM runSeq:: ATM :: +logKindFlag: ESMF_LOGKIND_MULTI_ON_ERROR diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a9bfa845ea..5a410adc4f 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -72,6 +72,7 @@ valid_vals_USE_CRTM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DO_LSM_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From 0ea005d719532490a1431d01affdfac99f977219 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Thu, 10 Mar 2022 09:24:37 -0700 Subject: [PATCH 131/203] Adding python utilities in preparation for providing a python workflow generation layer (#674) * Add python utility functions mirroring bash utils. * Add str_to_list utility, export varibales to environment Import env variables as numeric values to ease calculations. Address some of Christina's points. * Use dedent for multiline string formatting. * Bug fix with export_vars. * Bug fix with set_env_var * Add !join tag to yaml. * Bug fix with date conversion. * Beautify print_input_args. * Import stuff to __init__.py * Write dates in short form when HHMM = 0 * Clarify type conversions. * Add option to parse old style shell user config * Bug fix with handling same line comments * Minor bug fixes * Source complex config shell scripts instead of trying to parse. * Modify config shell script loading routine. * Update comments. * Addressing some of Christina's comments. * Removing set_bash/file_param. * Minor changes in unittests. * Remove unused config script parser. * Avoid passing os.environ as default. * Fix name of macos utils. * Add typical regional workflow resoultions to the test. * Add a copy of Externals.cfg to test. * More changes to address @gsketefian's suggestions. Co-authored-by: Daniel Shawul --- ush/python_utils/__init__.py | 20 ++ ush/python_utils/change_case.py | 25 ++ .../check_for_preexist_dir_file.py | 39 +++ ush/python_utils/check_var_valid_value.py | 21 ++ ush/python_utils/config_parser.py | 103 ++++++++ ush/python_utils/count_files.py | 16 ++ ush/python_utils/create_symlink_to_file.py | 49 ++++ ush/python_utils/define_macos_utilities.py | 39 +++ ush/python_utils/environment.py | 246 ++++++++++++++++++ ush/python_utils/filesys_cmds_vrfy.py | 35 +++ ush/python_utils/get_charvar_from_netcdf.py | 44 ++++ ush/python_utils/get_elem_inds.py | 30 +++ .../get_manage_externals_config_property.py | 54 ++++ ush/python_utils/interpol_to_arbit_CRES.py | 43 +++ ush/python_utils/print_input_args.py | 52 ++++ ush/python_utils/print_msg.py | 40 +++ ush/python_utils/process_args.py | 64 +++++ ush/python_utils/run_command.py | 25 ++ ush/python_utils/test_data/Externals.cfg | 38 +++ ush/python_utils/test_data/sample.nc | Bin 0 -> 10824 bytes ush/python_utils/test_data/var_defns.sh | 6 + ush/python_utils/test_python_utils.py | 125 +++++++++ 22 files changed, 1114 insertions(+) create mode 100644 ush/python_utils/__init__.py create mode 100644 ush/python_utils/change_case.py create mode 100644 ush/python_utils/check_for_preexist_dir_file.py create mode 100644 ush/python_utils/check_var_valid_value.py create mode 100644 ush/python_utils/config_parser.py create mode 100644 ush/python_utils/count_files.py create mode 100644 ush/python_utils/create_symlink_to_file.py create mode 100644 ush/python_utils/define_macos_utilities.py create mode 100644 ush/python_utils/environment.py create mode 100644 ush/python_utils/filesys_cmds_vrfy.py create mode 100644 ush/python_utils/get_charvar_from_netcdf.py create mode 100644 ush/python_utils/get_elem_inds.py create mode 100644 ush/python_utils/get_manage_externals_config_property.py create mode 100644 ush/python_utils/interpol_to_arbit_CRES.py create mode 100644 ush/python_utils/print_input_args.py create mode 100644 ush/python_utils/print_msg.py create mode 100644 ush/python_utils/process_args.py create mode 100644 ush/python_utils/run_command.py create mode 100644 ush/python_utils/test_data/Externals.cfg create mode 100644 ush/python_utils/test_data/sample.nc create mode 100644 ush/python_utils/test_data/var_defns.sh create mode 100644 ush/python_utils/test_python_utils.py diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py new file mode 100644 index 0000000000..1e4f6feed5 --- /dev/null +++ b/ush/python_utils/__init__.py @@ -0,0 +1,20 @@ +from .change_case import uppercase, lowercase +from .check_for_preexist_dir_file import check_for_preexist_dir_file +from .check_var_valid_value import check_var_valid_value +from .count_files import count_files +from .create_symlink_to_file import create_symlink_to_file +from .define_macos_utilities import define_macos_utilities +from .environment import str_to_date, date_to_str, str_to_type, type_to_str, list_to_str, \ + str_to_list, set_env_var, get_env_var, import_vars, export_vars +from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy +from .get_charvar_from_netcdf import get_charvar_from_netcdf +from .get_elem_inds import get_elem_inds +from .get_manage_externals_config_property import get_manage_externals_config_property +from .interpol_to_arbit_CRES import interpol_to_arbit_CRES +from .print_input_args import print_input_args +from .print_msg import print_info_msg, print_err_msg_exit +from .process_args import process_args +from .run_command import run_command +from .config_parser import cfg_to_shell_str, cfg_to_yaml_str, yaml_safe_load, \ + load_shell_config, load_config_file + diff --git a/ush/python_utils/change_case.py b/ush/python_utils/change_case.py new file mode 100644 index 0000000000..4fb46f94e4 --- /dev/null +++ b/ush/python_utils/change_case.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +def uppercase(str): + """ Function to convert a given string to uppercase + + Args: + str: the string + Return: + Uppercased str + """ + + return str.upper() + + +def lowercase(str): + """ Function to convert a given string to lowercase + + Args: + str: the string + Return: + Lowercase str + """ + + return str.lower() + diff --git a/ush/python_utils/check_for_preexist_dir_file.py b/ush/python_utils/check_for_preexist_dir_file.py new file mode 100644 index 0000000000..743f80fcc6 --- /dev/null +++ b/ush/python_utils/check_for_preexist_dir_file.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import os +from datetime import datetime +from .print_msg import print_info_msg, print_err_msg_exit +from .check_var_valid_value import check_var_valid_value +from .filesys_cmds_vrfy import rm_vrfy, mv_vrfy + +def check_for_preexist_dir_file(path, method): + """ Check for a preexisting directory or file and, if present, deal with it + according to the specified method + + Args: + path: path to directory + method: could be any of [ 'delete', 'rename', 'quit' ] + Returns: + None + """ + + check_var_valid_value(method, ['delete', 'rename', 'quit']) + + if os.path.exists(path): + if method == 'delete': + rm_vrfy(' -rf ', path) + elif method == 'rename': + now = datetime.now() + d = now.strftime("_%Y%m%d_%H%M%S") + new_path = path + d + print_info_msg(f''' + Specified directory or file already exists: + {path} + Moving (renaming) preexisting directory or file to: + {new_path}''') + mv_vrfy(path, new_path) + else: + print_err_msg_exit(f''' + Specified directory or file already exists + {path}''') + diff --git a/ush/python_utils/check_var_valid_value.py b/ush/python_utils/check_var_valid_value.py new file mode 100644 index 0000000000..0a033b7249 --- /dev/null +++ b/ush/python_utils/check_var_valid_value.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +from .print_msg import print_err_msg_exit + +def check_var_valid_value(var, values, err_msg=None): + """ Check if specified variable has a valid value + + Args: + var: the variable + values: list of valid values + err_msg: additional error message to print + Returns: + True: if var has valid value, exit(1) otherwise + """ + + if var not in values: + if err_msg is not None: + err_msg = f'The value specified in var = {var} is not supported.' + print_err_msg_exit(err_msg + f'{var} must be set to one of the following:\n {values}') + return True + diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py new file mode 100644 index 0000000000..f7d5a86eed --- /dev/null +++ b/ush/python_utils/config_parser.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import argparse +import yaml +import sys +import os +from textwrap import dedent + +from .environment import list_to_str, str_to_list +from .print_msg import print_err_msg_exit +from .run_command import run_command + +def load_shell_config(config_file): + """ Loads old style shell config files. + We source the config script in a subshell and gets the variables it sets + + Args: + config_file: path to config file script + Returns: + dictionary that should be equivalent to one obtained from parsing a yaml file. + """ + + # Save env vars before and after sourcing the scipt and then + # do a diff to get variables specifically defined/updated in the script + # Method sounds brittle but seems to work ok so far + code = dedent(f''' #!/bin/bash + (set -o posix; set) > /tmp/t1 + {{ . {config_file}; set +x; }} &>/dev/null + (set -o posix; set) > /tmp/t2 + diff /tmp/t1 /tmp/t2 | grep "> " | cut -c 3- + ''') + (_,config_str,_) = run_command(code) + lines = config_str.splitlines() + + #build the dictionary + cfg = {} + for l in lines: + idx = l.find("=") + k = l[:idx] + v = str_to_list(l[idx+1:]) + cfg[k] = v + return cfg + +def cfg_to_yaml_str(cfg): + """ Get contents of config file as a yaml string """ + + return yaml.dump(cfg, sort_keys=False, default_flow_style=False) + +def cfg_to_shell_str(cfg): + """ Get contents of yaml file as shell script string""" + + shell_str = '' + for k,v in cfg.items(): + v1 = list_to_str(v) + if isinstance(v,list): + shell_str += f'{k}={v1}\n' + else: + shell_str += f"{k}='{v1}'\n" + return shell_str + +def yaml_safe_load(file_name): + """ Safe load a yaml file """ + + try: + with open(file_name,'r') as f: + cfg = yaml.safe_load(f) + except yaml.YAMLError as e: + print_err_msg_exit(e) + + return cfg + +def join(loader, node): + """ Custom tag hangler to join strings """ + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + +yaml.add_constructor('!join', join, Loader=yaml.SafeLoader) + +def load_config_file(file_name): + """ Choose yaml/shell file based on extension """ + ext = os.path.splitext(file_name)[1][1:] + if ext == "sh": + return load_shell_config(file_name) + else: + return yaml_safe_load(file_name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=\ + 'Prints contents of yaml file as bash argument-value pairs.') + parser.add_argument('--cfg','-c',dest='cfg',required=True, + help='yaml or regular shell config file to parse') + parser.add_argument('--output-type','-o',dest='out_type',required=False, + help='output format: "shell": shell format, any other: yaml format ') + + args = parser.parse_args() + cfg = load_config_file(args.cfg) + + if args.out_type == 'shell': + print( cfg_to_shell_str(cfg) ) + else: + print( cfg_to_yaml_str(cfg) ) + diff --git a/ush/python_utils/count_files.py b/ush/python_utils/count_files.py new file mode 100644 index 0000000000..2239bf7eab --- /dev/null +++ b/ush/python_utils/count_files.py @@ -0,0 +1,16 @@ +import glob + +def count_files(ext,dirct='.'): + """ Function that returns the number of files in the specified directory + ending with the specified file extension + + Args: + ext: File extension string + dir: Directory to parse (default is current directory) + Returns: + int: Number of files + """ + + files = glob.glob(dirct + '/*.' + ext) + return len(files) + diff --git a/ush/python_utils/create_symlink_to_file.py b/ush/python_utils/create_symlink_to_file.py new file mode 100644 index 0000000000..1d078c9a8b --- /dev/null +++ b/ush/python_utils/create_symlink_to_file.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import os + +from .process_args import process_args +from .print_input_args import print_input_args +from .print_msg import print_err_msg_exit +from .check_var_valid_value import check_var_valid_value +from .filesys_cmds_vrfy import ln_vrfy + +def create_symlink_to_file(target,symlink,relative=True): + """ Create a symbolic link to the specified target file. + + Args: + target: target file + symlink: symbolic link to target file + relative: optional argument to specify relative symoblic link creation + Returns: + None + """ + + print_input_args(locals()) + + if target is None: + print_err_msg_exit(f''' + The argument \"target\" specifying the target of the symbolic link that + this function will create was not specified in the call to this function: + target = \"{target}\"''') + + if symlink is None: + print_err_msg_exit(f''' + The argument \"symlink\" specifying the target of the symbolic link that + this function will create was not specified in the call to this function: + symlink = \"{symlink}\"''') + + if not os.path.exists(target): + print_err_msg_exit(f''' + Cannot create symlink to specified target file because the latter does + not exist or is not a file: + target = \"{target}\"''') + + relative_flag="" + if relative: + RELATIVE_LINK_FLAG = os.getenv('RELATIVE_LINK_FLAG') + if RELATIVE_LINK_FLAG is not None: + relative_flag=f'{RELATIVE_LINK_FLAG}' + + ln_vrfy(f'-sf {relative_flag} {target} {symlink}') + diff --git a/ush/python_utils/define_macos_utilities.py b/ush/python_utils/define_macos_utilities.py new file mode 100644 index 0000000000..2bf996b996 --- /dev/null +++ b/ush/python_utils/define_macos_utilities.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import os + +from .print_msg import print_err_msg_exit +from .run_command import run_command +from .environment import set_env_var + +def check_darwin(cmd): + """ Check if darwin command exists """ + + (err,_,_) = run_command(f'command -v {cmd}') + if err != 0: + print_err_msg_exit(f''' + For Darwin-based operating systems (MacOS), the '{cmd}' utility is required to run the UFS SRW Application. + Reference the User's Guide for more information about platform requirements. + Aborting.''') + return True + +def define_macos_utilities(): + """ Set some environment variables for Darwin systems differently + The variables are: READLINK, SED, DATE_UTIL and LN_UTIL + """ + + if os.uname()[0] == 'Darwin': + if check_darwin('greadlink'): + set_env_var('READLINK','greadlink') + if check_darwin('gsed'): + set_env_var('SED','gsed') + if check_darwin('gdate'): + set_env_var('DATE_UTIL','gdate') + if check_darwin('gln'): + set_env_var('LN_UTIL','gln') + else: + set_env_var('READLINK','readlink') + set_env_var('SED','sed') + set_env_var('DATE_UTIL','date') + set_env_var('LN_UTIL','ln') + diff --git a/ush/python_utils/environment.py b/ush/python_utils/environment.py new file mode 100644 index 0000000000..e98deaad26 --- /dev/null +++ b/ush/python_utils/environment.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +import os +import inspect +import shlex +from datetime import datetime, date + +def str_to_date(s): + """ Get python datetime object from string. + It tests for only two formats used in RRFS: YYYYMMDD and YYYYMMDDHHMM + + Args: + s: a string + Returns: + datetime object or None + """ + try: + v = datetime.strptime(s, "%Y%m%d%H%M") + return v + except: + try: + v = datetime.strptime(s, "%Y%m%d") + return v + except: + return None + +def date_to_str(d,short=False): + """ Get string from python datetime object. + By default it converts to YYYYMMDDHHMM format unless + told otherwise with `short` or HHMM=0 + + Args: + d: datetime object + Returns: + string in YYYYMMDDHHMM or YYYYMMDD format + """ + if short or (d.hour == 0 and d.minute == 0): + v = d.strftime("%Y%m%d") + else: + v = d.strftime("%Y%m%d%H%M") + return v + +def str_to_type(s, just_get_me_the_string = False): + """ Check if the string contains a float, int, boolean, or just regular string. + This will be used to automatically convert environment variables to data types + that are more convenient to work with. If you don't want this functionality, + pass just_get_me_the_string = True + + Args: + s: a string + just_get_me_the_string: Set to True to return the string itself + Returns: + a float, int, boolean, date, or the string itself when all else fails + """ + s = s.strip('"\'') + if not just_get_me_the_string: + if s.lower() in ['true','yes','yeah']: + return True + elif s.lower() in ['false','no','nope']: + return False + v = str_to_date(s) + if v is not None: + return v + if s.isnumeric(): + return int(s) + try: + v = float(s) + return v + except: + pass + return s + +def type_to_str(v): + """ Given a float/int/boolean/date or list of these types, gets a string + representing their values + + Args: + v: a variable of the above types + Returns: + a string + """ + if isinstance(v,bool): + return ("TRUE" if v else "FALSE") + elif isinstance(v,int) or isinstance(v,float): + pass + elif isinstance(v,date): + return date_to_str(v) + elif v is None: + return '' + return str(v) + +def list_to_str(v, oneline=False): + """ Given a string or list of string, construct a string + to be used on right hand side of shell environement variables + + Args: + v: a string/number, list of strings/numbers, or null string('') + Returns: + A string + """ + if isinstance(v,str): + return v + if isinstance(v, list): + v = [type_to_str(i) for i in v] + if oneline or len(v) <= 4: + shell_str = f'( "' + '" "'.join(v) + '" )' + else: + shell_str = f'( \\\n"' + '" \\\n"'.join(v) + '" \\\n)' + else: + shell_str = f'{type_to_str(v)}' + + return shell_str + +def str_to_list(v): + """ Given a string, construct a string or list of strings. + Basically does the reverse operation of `list_to_string`. + + Args: + v: a string + Returns: + a string, list of strings or null string('') + """ + + if not isinstance(v,str): + return v + v = v.strip() + if not v: + return None + if v[0] == '(': + v = v[1:-1] + tokens = shlex.split(v) + lst = [] + for itm in tokens: + itm = itm.strip() + if itm == '': + continue + # bash arrays could be stored with indices ([0]=hello ...) + if "=" in itm: + idx = itm.find("=") + itm = itm[idx+1:] + lst.append(str_to_type(itm)) + return lst + else: + return str_to_type(v) + +def set_env_var(param,value): + """ Set an environment variable + + Args: + param: the variable to set + value: either a string, list of strings or None + Returns: + None + """ + + os.environ[param] = list_to_str(value) + +def get_env_var(param): + """ Get the value of an environement variable + + Args: + param: the environement variable + Returns: + Returns either a string, list of strings or None + """ + + if not param in os.environ: + return None + else: + value = os.environ[param] + return str_to_list(value) + +def import_vars(dictionary=None, target_dict=None, env_vars=None): + """ Import all (or select few) environment/dictionary variables as python global + variables of the caller module. Call this function at the beginning of a function + that uses environment variables. + + Note that for read-only environmental variables, calling this function once at the + beginning should be enough. However, if the variable is modified in the module it is + called from, the variable should be explicitly tagged as `global`, and then its value + should be exported back to the environment with a call to export_vars() + + import_vars() # import all environment variables + global MY_VAR, MY_LIST_VAR + MY_PATH = "/path/to/somewhere" + MY_LIST_VAR.append("Hello") + export_vars() # these exports all global variables + + There doesn't seem to an easier way of imitating the shell script doing way of things, which + assumes that everything is global unless specifically tagged local, while the opposite is true + for python. + + Args: + dictionary: source dictionary (default=os.environ) + target_dict: target dictionary (default=caller module's globals()) + env_vars: list of selected environement/dictionary variables to import, or None, + in which case all environment/dictionary variables are imported + Returns: + None + """ + if dictionary is None: + dictionary = os.environ + + if target_dict is None: + target_dict = inspect.stack()[1][0].f_globals + + if env_vars is None: + env_vars = dictionary + else: + env_vars = { k: dictionary[k] if k in dictionary else None for k in env_vars } + + for k,v in env_vars.items(): + target_dict[k] = str_to_list(v) + +def export_vars(dictionary=None, source_dict=None, env_vars=None): + """ Export all (or select few) global variables of the caller module's + to either the environement/dictionary. Call this function at the end of + a function that updates environment variables. + + Args: + dictionary: target dictionary to set (default=os.environ) + source_dict: source dictionary (default=caller modules globals()) + env_vars: list of selected environement/dictionary variables to export, or None, + in which case all environment/dictionary variables are exported + Returns: + None + """ + if dictionary is None: + dictionary = os.environ + + if source_dict is None: + source_dict = inspect.stack()[1][0].f_globals + + if env_vars is None: + env_vars = source_dict + else: + env_vars = { k: source_dict[k] if k in source_dict else None for k in env_vars } + + for k,v in env_vars.items(): + # skip functions and other unlikely variable names + if callable(v): + continue + if not k or k.islower() or k[0] == '_': + continue + dictionary[k] = list_to_str(v) + diff --git a/ush/python_utils/filesys_cmds_vrfy.py b/ush/python_utils/filesys_cmds_vrfy.py new file mode 100644 index 0000000000..67f1f45f9c --- /dev/null +++ b/ush/python_utils/filesys_cmds_vrfy.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import os +from .print_msg import print_err_msg_exit + +def cmd_vrfy(cmd, *args): + """ Execute system command + + Args: + cmd: the command + *args: its arguments + Returns: + Exit code + """ + + for a in args: + cmd += ' ' + str(a) + ret = os.system(cmd) + if ret != 0: + print_err_msg_exit(f'System call "{cmd}" failed.') + return ret + +def cp_vrfy(*args): + return cmd_vrfy('cp', *args) +def mv_vrfy(*args): + return cmd_vrfy('mv', *args) +def rm_vrfy(*args): + return cmd_vrfy('rm', *args) +def ln_vrfy(*args): + return cmd_vrfy('ln', *args) +def mkdir_vrfy(*args): + return cmd_vrfy('mkdir', *args) +def cd_vrfy(*args): + return os.chdir(*args) + diff --git a/ush/python_utils/get_charvar_from_netcdf.py b/ush/python_utils/get_charvar_from_netcdf.py new file mode 100644 index 0000000000..c85e42e358 --- /dev/null +++ b/ush/python_utils/get_charvar_from_netcdf.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import os +from .print_msg import print_err_msg_exit +from .run_command import run_command + +def get_charvar_from_netcdf(nc_file, nc_var_name): + """ Searches NetCDF file and extract a scalar variable + + Args: + nc_file: Path to netCDF file + nc_var_name: name of the scalar variable + Returns: + value of the variable + """ + + SED = os.getenv('SED') + + cmd = f"ncdump -v {nc_var_name} {nc_file} | \ + {SED} -r -e '1,/data:/d' \ + -e '/^[ ]*'{nc_var_name}'/d' \ + -e '/^}}$/d' \ + -e 's/.*\"(.*)\".*/\\1/' \ + -e '/^$/d' \ + " + (ret,nc_var_value,_) = run_command(cmd) + + if ret != 0: + print_err_msg_exit(f''' + Attempt to extract the value of the NetCDF variable spcecified by nc_var_name + from the file specified by nc_file failed: + nc_file = \"{nc_file}\" + nc_var_name = \"{nc_var_name}\"''') + + if nc_var_value is None: + print_err_msg_exit(f''' + In the specified NetCDF file (nc_file), the specified variable (nc_var_name) + was not found: + nc_file = \"{nc_file}\" + nc_var_name = \"{nc_var_name}\" + nc_var_value = \"{nc_var_value}\"''') + + return nc_var_value + diff --git a/ush/python_utils/get_elem_inds.py b/ush/python_utils/get_elem_inds.py new file mode 100644 index 0000000000..fd4ea2cee0 --- /dev/null +++ b/ush/python_utils/get_elem_inds.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from .change_case import lowercase +from .check_var_valid_value import check_var_valid_value + +def get_elem_inds(arr, match, ret_type): + """ Function that returns indices of elements of array + that match a given string + + Args: + arr: the list + match: element to match (case insenensitive) + ret_type: the return type can be any of [ 'first', 'last', 'all' ] + Returns: + A list of indices + """ + ret_type = lowercase(ret_type) + check_var_valid_value(ret_type, ['first', 'last', 'all']) + + if ret_type == 'first': + for i,e in enumerate(arr): + if e == match: + return i + elif ret_type == 'last': + for i in range(len(arr)-1, -1, -1): + if arr[i] == match: + return i + else: + return [i for i,e in enumerate(arr) if e == match] + diff --git a/ush/python_utils/get_manage_externals_config_property.py b/ush/python_utils/get_manage_externals_config_property.py new file mode 100644 index 0000000000..c537a88244 --- /dev/null +++ b/ush/python_utils/get_manage_externals_config_property.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import os +import configparser + +from .print_msg import print_err_msg_exit + +def get_manage_externals_config_property(externals_cfg_fp, external_name, property_name): + """ + This function searches a specified manage_externals configuration file + and extracts from it the value of the specified property of the external + with the specified name (e.g. the relative path in which the external + has been/will be cloned by the manage_externals utility). + + Args: + + externals_cfg_fp: + The absolute or relative path to the manage_externals configuration + file that will be searched. + + external_name: + The name of the external to search for in the manage_externals confi- + guration file specified by externals_cfg_fp. + + property_name: + The name of the property whose value to obtain (for the external spe- + cified by external_name). + + Returns: + The property value + """ + + if not os.path.exists(externals_cfg_fp): + print_err_msg_exit(f''' + The specified manage_externals configuration file (externals_cfg_fp) + does not exist: + externals_cfg_fp = \"{externals_cfg_fp}\"''') + + config = configparser.ConfigParser() + config.read(externals_cfg_fp) + + if not external_name in config.sections(): + print_err_msg_exit(f''' + In the specified manage_externals configuration file (externals_cfg_fp), + the specified property (property_name) was not found for the the speci- + fied external (external_name): + externals_cfg_fp = \"{externals_cfg_fp}\" + external_name = \"{external_name}\" + property_name = \"{property_name}\"''') + else: + return config[external_name][property_name] + + return None + diff --git a/ush/python_utils/interpol_to_arbit_CRES.py b/ush/python_utils/interpol_to_arbit_CRES.py new file mode 100644 index 0000000000..f6ec889b24 --- /dev/null +++ b/ush/python_utils/interpol_to_arbit_CRES.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +def interpol_to_arbit_CRES(RES, RES_array, prop_array): + """ Function to interpolate (or extrapolate) a grid cell size-dependent property + to an arbitrary cubed-sphere resolution using arrays that specify a set of property + values for a corresponding set of resolutions + + Args: + RES: The cubed-sphere resolution at which to find the value of a property. + This is in units of number of cells (in either of the two horizontal + directions) on any one of the tiles of a cubed-sphere grid. + + RES_array: The name of the array containing the cubed-sphere resolutions for + which corresponding property values are given (in prop_array). These + are assumed to be given from smallest to largest. + + prop_array: The name of the array containing the values of the property corres- + ponding to the cubed-sphere resolutions in RES_array. + Returns: + Interpolated (extrapolated) property value + """ + + num_valid_RESes = len(RES_array) + i_min = 0 + i_max = num_valid_RESes - 1 + + if RES <= RES_array[i_min]: + prop = prop_array[i_min] + elif RES > RES_array[i_max]: + prop = prop_array[i_max] + else: + for i in range(0,num_valid_RESes-1): + if RES > RES_array[i] and RES <= RES_array[i+1]: + RES1 = RES_array[i] + RES2 = RES_array[i+1] + prop1 = prop_array[i] + prop2 = prop_array[i+1] + m_slope = (prop2 - prop1) / (RES2 - RES1) + y_intcpt = (RES2 * prop1 - RES1 * prop2) / (RES2 - RES1) + prop = m_slope * RES + y_intcpt + + return prop + diff --git a/ush/python_utils/print_input_args.py b/ush/python_utils/print_input_args.py new file mode 100644 index 0000000000..4f168a612f --- /dev/null +++ b/ush/python_utils/print_input_args.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import os +import inspect +from textwrap import dedent + +from .change_case import lowercase +from .print_msg import print_info_msg +from .environment import import_vars + +def print_input_args(valid_args): + """ Prints function arguments for debugging purposes + + Args: + valid_args: dictionary of arg-value pairs + Returns: + Number of printed arguments + """ + + # get verbosity from environment + IMPORTS = ["DEBUG"] + import_vars(env_vars=IMPORTS) + + if list(valid_args.keys())[0] == '__unset__': + valid_arg_names = {} + else: + valid_arg_names = valid_args + num_valid_args = len(valid_arg_names) + + filename = inspect.stack()[1].filename + function = inspect.stack()[1].function + filename_base = os.path.basename(filename) + + if num_valid_args == 0: + msg = dedent(f''' + No arguments have been passed to function {function} in script {filename_base} located + + \"{filename}\"''') + else: + msg = dedent(f''' + The arguments to function {function} in script {filename_base} located + + \"{filename}\" + + have been set as follows:\n\n''') + + for k,v in valid_arg_names.items(): + msg = msg + f' {k}="{v}"\n' + + print_info_msg(msg,verbose=DEBUG) + return num_valid_args + diff --git a/ush/python_utils/print_msg.py b/ush/python_utils/print_msg.py new file mode 100644 index 0000000000..75858e5613 --- /dev/null +++ b/ush/python_utils/print_msg.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import traceback +import sys +from textwrap import dedent + +def print_err_msg_exit(error_msg="",stack_trace=True): + """Function to print out an error message to stderr and exit. + It can optionally print the stack trace as well. + + Args: + error_msg : error message to print + stack_trace : set to True to print stack trace + Returns: + None + """ + if stack_trace: + traceback.print_stack(file=sys.stderr) + + msg_footer='\nExiting with nonzero status.' + print(dedent(error_msg) + msg_footer, file=sys.stderr) + sys.exit(1) + +def print_info_msg(info_msg,verbose=True): + """ Function to print information message to stdout, when verbose + is set to True. It does proper "dedentation" that is needed for readability + of python code. + + Args: + info_msg : info message to print + verbose : set to False to silence printing + Returns: + True: if message is successfully printed + """ + + if verbose == True: + print(dedent(info_msg)) + return True + return False + diff --git a/ush/python_utils/process_args.py b/ush/python_utils/process_args.py new file mode 100644 index 0000000000..0baf0d4f9b --- /dev/null +++ b/ush/python_utils/process_args.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 + +from textwrap import dedent +from .check_var_valid_value import check_var_valid_value + +def process_args(valid_args, **kwargs): + """ Function to process a list of variable name-value pairs. + It checks whether each argument is a valid argument or not. + + Args: + valid_args: List of valid arguments + **kwargs: keyword arguments + Returns: + A dictionary of all valid (arg,value) pairs + """ + + if valid_args[0] == '__unset__': + valid_arg_names = [] + else: + valid_arg_names = valid_args + num_valid_args = len(valid_arg_names) + num_arg_val_pairs = len(kwargs) + + if num_arg_val_pairs > num_valid_args: + print_err_msg_exit(f''' + The number of argument-value pairs specified on the command line (num_- + arg_val_pairs) must be less than or equal to the number of valid argu- + ments (num_valid_args) specified in the array valid_arg_names: + num_arg_val_pairs = {num_arg_val_pairs} + num_valid_args = {num_valid_args} + valid_arg_names = ( {valid_arg_names_str})''') + + if num_valid_args == 0: + return None + + values_args = [None] * num_valid_args + + for i,a in enumerate(valid_args): + if a is None: + print_err_msg_exit(f''' + The list of valid arguments (valid_arg_names) cannot contain empty elements, + but the element with index i={i} is empty: + valid_arg_names = ( {valid_arg_names}) + valid_arg_names[{i}] = \"{valid_arg_names[i]}\"''') + + for arg_name,arg_value in kwargs.items(): + err_msg=dedent(f''' + The specified argument name (arg_name) in the current argument-value + pair (arg_val_pair) is not valid: + arg_name = \"{arg_name}\" + arg_val = \"{arg_value}\"\n''') + check_var_valid_value(arg_name, valid_arg_names, err_msg) + + idx = valid_arg_names.index(arg_name) + if values_args[idx] is not None: + print_err_msg_exit(f''' + The current argument has already been assigned a value: + arg_name = \"{arg_name}\" + key_value_pair = {kwargs} + Please assign values to arguments only once on the command line.''') + values_args[idx] = arg_value + + return dict(zip(valid_args,values_args)) + diff --git a/ush/python_utils/run_command.py b/ush/python_utils/run_command.py new file mode 100644 index 0000000000..70978bd789 --- /dev/null +++ b/ush/python_utils/run_command.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import subprocess + +def run_command(cmd): + """ Run system command in a subprocess + + Args: + cmd: command to execute + Returns: + Tuple of (exit code, std_out, std_err) + """ + proc = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + universal_newlines=True) + + std_out, std_err = proc.communicate() + + # strip trailing newline character + return (proc.returncode, + std_out.rstrip('\n'), + std_err.rstrip('\n')) + diff --git a/ush/python_utils/test_data/Externals.cfg b/ush/python_utils/test_data/Externals.cfg new file mode 100644 index 0000000000..43e6e7b9b4 --- /dev/null +++ b/ush/python_utils/test_data/Externals.cfg @@ -0,0 +1,38 @@ +[regional_workflow] +protocol = git +repo_url = https://github.com/ufs-community/regional_workflow +# Specify either a branch name or a hash but not both. +#branch = develop +hash = 20a149d +local_path = regional_workflow +required = True + +[ufs_utils] +protocol = git +repo_url = https://github.com/ufs-community/UFS_UTILS +# Specify either a branch name or a hash but not both. +#branch = develop +hash = f30740e +local_path = src/UFS_UTILS +required = True + +[ufs-weather-model] +protocol = git +repo_url = https://github.com/ufs-community/ufs-weather-model +# Specify either a branch name or a hash but not both. +#branch = develop +hash = 805421d +local_path = src/ufs-weather-model +required = True + +[UPP] +protocol = git +repo_url = https://github.com/NOAA-EMC/UPP +# Specify either a branch name or a hash but not both. +#branch = develop +hash = a49af05 +local_path = src/UPP +required = True + +[externals_description] +schema_version = 1.0.0 diff --git a/ush/python_utils/test_data/sample.nc b/ush/python_utils/test_data/sample.nc new file mode 100644 index 0000000000000000000000000000000000000000..9e31e16a3d622e35de5e5f753c96dd95ae118665 GIT binary patch literal 10824 zcmb`M30PFs-^T~V4KWQh!zIUDnFY|%6c?Ct&X}oKnoD3Nk}!(0$>LI)|(T2jgkdao9az8wa}^fr%3i5@@s@woWd zz)8_lqis{oT?4xXo4W-EJA#6H1O@jr2Y2t?!_m8&!<-O3K0Y=&(cFJ}vbh7k->~wi zx)xo723^k^N1gPf$pm&+WMD^hJzD32~_~KOt(Wj+pLe=>Pio zTS%|b@o8~s=`m#O@ATK2oB*n3eZEz8%tMBUgSlU49lYmt;^!60BM8&mh zs-K2wu?fksDbZ;aAu!dbeZ)VfJsYNK#Uv|(|Ecz7F|pHD?$ z{wH-+RX3Z+hlG^>mk-_(f+kM@SySwN6XK?0YBMAHaY>}Drn;G?F7uD*-U?IR(^8Wu z5Pui@+6h5wrmso2&$PJ27;|cJY^(}tjk>F@^H0P(SLu$4qlo{V?s}DTCeRvV{(#aB zTe0I2^hCn`7yT92`zQ40;Tm;ZDM`tdYiD&Iyy>(1)>W_eq^f59?b!J7=?UprMPg&j ziRlT?$EKK*+~!oQ!?ETV;~Mwtrt5h9ac@R6UTa#fs(()BIO^|Vr{-SzHN#};_xX?8 zdElr+Qk&*>C0;VIcQdnC9FE|!MApaQ5W(ece8)um@8Co0^txWjrz$?TsF=m$aP+aT zu9;cY_{+73Z{=zA7ans2>#^z&X8n`>%k_v4NqNHH&u1~RulTh$nd8sAFY)2|D=PV= zx}M1n2kZ90zwfvIAn{{+gn@qr{_b@pR@ztkosF0o>qzB<3V$y@M10)zv*3q=|1ZT; ze3bpB$5`v4Vvz(0gZNfBHTdaPJ)Tv1%&^3Lp&P zJK{8S%In8?;K;q;XXy5;@)hWIi}3K`D&9i;^aCfrzYKn-uD{CPN5GE~|;r)j>;m;D~$RBmSbt41d!c4wzT=Lx^8CvRkG7*}9&pcs>L@ zZ!FREhZ6tx%XPsIf&DDqexlpWy2k}GGwg5H>+`Z8er>xyD&jfgf*#LG-nkF^dZx3Bnz*N0_hW`%|udR`#@y$e5s|4nK$1O8%P#mC6kY~tTPQNjbC zdkOrc3j2s^k3&m1HETZq@{b~Jdfdokatcn&;1T!c)4b}!NcD|@G734e53d|CN;?ZKH`u1 zB~|hj{vJVm{=^&kJ*aq^KUMtUFZ2_y)~_uWnVI}Gcn|R(tj;lb=syTOir;Fh93kNuil3dS=j$Wlk9WNTJ{f$z;we7W?N^(hPZ58jO?f3> zr1PoxPOlbil>G!FUuTH_@Wcz?>%sncm0!%UO!0m#RQ-%H>gT-TEy0yM>?8hp$feRh zUhkmNhu6-t}dTwsi&-Xe%>R*_D+Pjj>Ce^>H9u}Wv4bhK&Mtp7% z|3&sP9{5xjtiK0c3FAy`7h(w=tm7L~Sp)c6$IyQZe3;jheR<%3-=#Knr)v`8L-}&S zhn6euu&L|`! zgln8Te38Rf{)tHq^kYeL;71d!h-q|N5@f zvah)q@`XG8Fp6ou$I3!b54$^6%~vIX#5e1?hud%1rM5N8)h{xOz3Fy`g@ex>yF%sp zuHmnP__jX%xcxiS^Ul7m5af3r?7N|7clKLUZ)}ti|6at~zCO+Em;aF3-ixjt(1TtH zb3k|gnjqE$y1y{&_anZ?=Qp_h%gItZ*w59?X<|cR--P!I`u1Qp)G5{s{Hy%s#D|i< zpLLg-=NnfL`U^hxBY*SCJZucs_dI_+J_Cq8CS#D|AOaC;%@|HGcH0Q8H0W>z7^XU8Ky(>y2gQ>*r$CBul1`1vTe@7^x8 zQ$1Xrpx3*oxG)Ft+44;u8-efP`?#FaKZ^M0PjPPFyi00l+PXT_bvrMaOobl2zcu7B z^4*zh#OGP!C*gj|z8dxXMK4!7|12l!zc3T^xvR@2M!LQ-_;JKf`AFmTapY@h6x6i*Wwad@DTJ6trZge<2GhlzY-*$#qvDL6Y ziTF9LPr3aC)Z4eEF8>0Da{=t9BEKu`*)-RQ%_cRdKIvA*IlpuSwXPrdaO^+L7%MoAj+3x~Dj_E^N{hf%KPsSYQ6^sGQU-{@DDodmzb&_9FtrCIUZ zPW$1X+PeG}xM`2%ISzhaeXvVvx6is7=N8lc&T|Rx&kE_o zpi6Aa(d%t4@hiF~ay$Q8YIhsE8ktOoi;9Xh)TDFBRE-^_{gTH-gZy2X0e;QoVchO; zNa^*h%V&YR(Cv0>X1xE@><1NZ?3YTG5}*CG7q@pTm9n0Vb^GF5*v(qSWkTZ zukUi(?Xyz0>?wYa%<>>6+7#4J^DFh(O2lWq(?o;x=VX9?Z}@F)`~J9;ZD)x)@Upc*oF_NBrTiS={zT zl#~`t+`Q?~yug18zEAtP%me+|k$OMwCBBgKd~#mO;6CEV@M5ht^dAGi@LUW#1^pWg zJqL(C+I}OqNna^@E)_qROqwt1f%?V2#rNzQ^lS{T+HaH;5`Sz`3vS!rLCU_9#J|sH zX|2JNzsGvdWLKeQqY?iPi2rcaNp9P5MoN(A*=zrdWE zbgULT1O4k2uK3f$pOvq3+tSlg4qqcKI8EAE(*8kY8U1yU_^($*aa-DtQjYsZ;QCqyjy=$G_`%a`9r#s7 zeoKhIg!?tyq`^|UPl?Z$6l?RLA9F(3=c{M2_0W@fUeE6p;!8YpxovzMDJQ$cr{}X2 z51US}{LL&2{Y6EOLGihUQ|Gr4CESMRGcwg*d{XY#79z*soTgoFwh8fBm#N$Tk@%~) zf42?lC*=!mL=k*ljDzfZZwpiPTrHlz5dR(a|2C&d%GpQ6haQu*4F00$!>;7tW1FGB zTEF~8{P){G;kF*HNjX16oJe(O%b@2|#OJM+=`0)hU9ZJ3Y*8o!3jbWU=_5 zz@aUN{T~tkffHX*@yXWXSz3qqpW3bDwl+ahzS&f0@b?|JTh;;J{GyHRhMskiRsNRx z5dTX_OK$T!Eal3jq7ZtZSvJOf^0l398S*CH(d)moG4a2?@;kSAuaWYd)8epiv4;Js z^g%t>>pX-Vrur$UT2D)x5&t{t&3b2xlpF7f1K{^TPd(^qw$qPk;MX`!G?1@S{|vmE zE`7tT*Xm2T?XGw)GD|xE{vqs>Kli)H4na?*VgC{0YnkG>wG{R6?lghE(mX}Q5OMlb}Pa<^XYAdS`u&R{ab1$zW$4Yxb?kLQXZNwc7T`A-^=X|bxfbb z=7C@ATQ#4Qb|c=0_?_ia7LF2Iz+;Y*{h-JE_AQm)IXCrq_9VX1()YP_T|Ft~XJQk0 z^rFOr&(H^N-c;*jh7q5>#6OU?h+CI^E9KEVu@OAx2I+#HI=34#d{;~}c&7OGpXOHT zciCBNbU3IsB?sT=e0O#R`d>8S`MBagdy-qJ-wS7n4dreP`*Ya}@h{pH$-YECW*GVh z6W@67SZ`D%@N zC?0T^?w!TF^LjmxRQ$@N+&VN+%G^O>9sDKx51=3YjtrxF4Kd%y?`Y!v?9;eaL`%7D zrC6I`(u!c8>N99tu$m96@lnJ#eP%1S_IXIkw>yf}sJ9~6uaEu>I$NJ@gZ(9jo*3er zH~N8FyWWv{+8 z(&yQ0g1FsK1e3x5sKPA&IinkHZQt-9l@6(U%WSA?&G(Z@{r{a|PkH5jK ztzm!ie6ifOSi1^;eZhCUxSP#^{WK%L(}{1{uRgan`$)>@C&XLeufbpHcY9GkRnJKV zKa2QQonGcv-I!zk(ZYrMkulyP2A`zdd`@5QWbN#N|^0k+gEtZPKId16%{Y{Wx zbIy0lev%QNY~tH>ewthU6@vX@^Gw@9ZI2h(qG1kz(o%r_Owc(ZzaKHRpj95V0akN1Eecp7d{)zyELH2jy zly&(1CvG_yBem-vi}~el>Me<9EMY!PZ!yeuVuWFT5AhwdYH>@!M5$ek5c6(26z>n- zyV*&0oOq+(_bVRHOD(zmq;~0;m{$NEdRn61OalYy*@YNs)WZ?tJNf2w%lf8LJ3Cs; z%P>h_dP}0dX^=| z82)}ve89fPxn)KJspV}F8NS7`8F(A`tvRz9)+RBkNMA3`5pQAbxFr#Kwwx5RQ4h@! zpCHWtt9!Lm`>~M*|CQojy}&IK5~Q|%k$4HVA40x5Lx17K0_K692&3LE6K`z``_Dcj zwH52d%nb0b-x>Dh+G}b(s>YWq{)#uZ(DSRs>%@z&jo8aBSRW7eU&FS-{wO0~-w|)a z{@K#Mt<>hEix;53CB733f3vQnFzmI2@jS5fy5c`XeTE*x^Sb6@24au3MdF@6bVKZN z)$gN>di$Apd&WE5(hbi;Qbver$X9FV4}ktd5%FvZ#nY&VUx^Rw@(Z_g#QpVz>mnWX z_6T^|4-{;8o^7IfGvfJ&;_sw#ODp^yF|MUZLp<9-Q#a^eF|?PuUmIys1NBhmO?=P> z2e?szz8m}vxV}kheXfXj@USg= z!ruimEev}zF|x>^Pqg3xS2d}1KO`n0Hi0;LBmUFk%hdha06-YzZyTI4N4}j0 ze2w2PEU{uzcm>}F_EV;eQO^_l>+vspRPi4N@PIRTKG61vh=XlA^pO4OFaDcNNBmue z{?3a3sVxsUHAZUw7epM{M?Cw$-(>Ae7K!zDfL?E9HsZU#^eYeeV2IRw3&ceKEb0~6 z7xfu;b2@vH{59g)mG~YHUFQLZ9+O(Fbm7i0Q7mOB@-=?n2kb%EA87E8DW3G?e=9w= z+#(iQ9N?YcBbH31XH&vu_}hp0$0p6@0ow;l&-atW_yULQLH401I--O6ecY+{W0{lq zp5h)4*zlzEl)WsThy9-5`yxImf7EBVcdPb$bXkAJ<9S*D7|)jrM6`dF41vE+=ovFa zung2gf5ZMD;(M8o@qjnFOV6jX#5nld3psE>k2B*Z_596c@J}f|F^C5&>M1=Z5(G9x zw6~Q{fVXwsO3woY53ic9%0>_$QgB44OAISBeY_KH&R>2Jg{Qt{tx z;sL4qq$fW~jDc;^gB*oEdh#=dS0R)hFRHh(#P@ce;sKK%lb&ta;#uhF4So>xbd2t+ z`nMuqiXX3d{Jt3w(@=V{JBw!#Pt=+mg!lyPva|ij@1RUtuN6O0@fYFmm|W>u?k6Bn z#d9#~t$n>uSt8 zoR%KUZxNJRXc`JVjb)1ZJ))X^+RCxMl&>e;(vvnqJOdUFW92Z|_i6EwnjZ$K_ Date: Fri, 11 Mar 2022 14:47:34 -0500 Subject: [PATCH 132/203] Fix crontab newline issue on wcoss_dell_p3 (#693) * clean up wcoss_cray and unnecessary modules on wcoss * Fix a new line issue on wcoss_dell_p3 * Remove docs from regional_workflow --- docs/UsersGuide/Makefile | 19 -- docs/UsersGuide/README | 29 -- docs/UsersGuide/build/.gitignore | 4 - docs/UsersGuide/make.bat | 35 --- docs/UsersGuide/requirements.txt | 1 - docs/UsersGuide/source/Acknowledgement.rst | 20 -- docs/UsersGuide/source/Chapter1.rst | 27 -- docs/UsersGuide/source/Chapter2.rst | 264 ------------------ docs/UsersGuide/source/Chapter3.rst | 170 ----------- docs/UsersGuide/source/Chapter4.rst | 261 ----------------- docs/UsersGuide/source/Introduction.rst | 41 --- docs/UsersGuide/source/_static/custom.css | 6 - docs/UsersGuide/source/_templates/.gitignore | 4 - docs/UsersGuide/source/conf.py | 202 -------------- docs/UsersGuide/source/index.rst | 17 -- docs/UsersGuide/source/prolog.inc | 4 - .../tasks/wcoss_cray/get_extrn_ics.local | 11 - .../tasks/wcoss_cray/get_extrn_lbcs.local | 11 - modulefiles/tasks/wcoss_cray/get_obs.local | 9 - modulefiles/tasks/wcoss_cray/make_grid.local | 15 - modulefiles/tasks/wcoss_cray/make_ics.local | 17 -- modulefiles/tasks/wcoss_cray/make_lbcs.local | 17 -- modulefiles/tasks/wcoss_cray/make_orog.local | 12 - .../tasks/wcoss_cray/make_sfc_climo.local | 12 - modulefiles/tasks/wcoss_cray/run_fcst.local | 15 - modulefiles/tasks/wcoss_cray/run_post.local | 18 -- modulefiles/tasks/wcoss_cray/run_vx.local | 13 - .../tasks/wcoss_dell_p3/get_extrn_ics.local | 3 - .../tasks/wcoss_dell_p3/get_extrn_lbcs.local | 3 - .../tasks/wcoss_dell_p3/run_post.local | 7 - ush/generate_FV3LAM_wflow.sh | 2 +- 31 files changed, 1 insertion(+), 1268 deletions(-) delete mode 100644 docs/UsersGuide/Makefile delete mode 100644 docs/UsersGuide/README delete mode 100644 docs/UsersGuide/build/.gitignore delete mode 100644 docs/UsersGuide/make.bat delete mode 100644 docs/UsersGuide/requirements.txt delete mode 100644 docs/UsersGuide/source/Acknowledgement.rst delete mode 100644 docs/UsersGuide/source/Chapter1.rst delete mode 100644 docs/UsersGuide/source/Chapter2.rst delete mode 100644 docs/UsersGuide/source/Chapter3.rst delete mode 100644 docs/UsersGuide/source/Chapter4.rst delete mode 100644 docs/UsersGuide/source/Introduction.rst delete mode 100644 docs/UsersGuide/source/_static/custom.css delete mode 100644 docs/UsersGuide/source/_templates/.gitignore delete mode 100644 docs/UsersGuide/source/conf.py delete mode 100644 docs/UsersGuide/source/index.rst delete mode 100644 docs/UsersGuide/source/prolog.inc delete mode 100644 modulefiles/tasks/wcoss_cray/get_extrn_ics.local delete mode 100644 modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local delete mode 100644 modulefiles/tasks/wcoss_cray/get_obs.local delete mode 100644 modulefiles/tasks/wcoss_cray/make_grid.local delete mode 100644 modulefiles/tasks/wcoss_cray/make_ics.local delete mode 100644 modulefiles/tasks/wcoss_cray/make_lbcs.local delete mode 100644 modulefiles/tasks/wcoss_cray/make_orog.local delete mode 100644 modulefiles/tasks/wcoss_cray/make_sfc_climo.local delete mode 100644 modulefiles/tasks/wcoss_cray/run_fcst.local delete mode 100644 modulefiles/tasks/wcoss_cray/run_post.local delete mode 100644 modulefiles/tasks/wcoss_cray/run_vx.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/run_post.local diff --git a/docs/UsersGuide/Makefile b/docs/UsersGuide/Makefile deleted file mode 100644 index 69fe55ecfa..0000000000 --- a/docs/UsersGuide/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/UsersGuide/README b/docs/UsersGuide/README deleted file mode 100644 index 30617076c6..0000000000 --- a/docs/UsersGuide/README +++ /dev/null @@ -1,29 +0,0 @@ -Steps to build and use the Sphinx documentation tool: - -1) Get Sphinx and sphinxcontrib-bibtex installed on your desktop from - http://www.sphinx-doc.org/en/master/usage/installation.html - https://sphinxcontrib-bibtex.readthedocs.io/en/latest/quickstart.html#installation - -2) Create a Sphinx documentation root directory: - % mkdir docs - % cd docs - -3) Initialize your Sphinx project (set up an initial directory structure) using - % sphinx-quickstart - - See http://www.sphinx-doc.org/en/master/usage/quickstart.html or - https://sphinx-rtd-tutorial.readthedocs.io/en/latest/sphinx-quickstart.html - - for help. You can answer (ENTER) to most of the questions. - -To build html: - -From the directory above source and build, the sphinx project directory: - -make html - -Sphinx uses Latex to export the documentation as a PDF file. To build pdf: - -make latexpdf - -It will generate a PDF file in ./build/latex/.pdf diff --git a/docs/UsersGuide/build/.gitignore b/docs/UsersGuide/build/.gitignore deleted file mode 100644 index 5e7d2734cf..0000000000 --- a/docs/UsersGuide/build/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/docs/UsersGuide/make.bat b/docs/UsersGuide/make.bat deleted file mode 100644 index 543c6b13b4..0000000000 --- a/docs/UsersGuide/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% - -:end -popd diff --git a/docs/UsersGuide/requirements.txt b/docs/UsersGuide/requirements.txt deleted file mode 100644 index ef36addc62..0000000000 --- a/docs/UsersGuide/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinxcontrib-bibtex diff --git a/docs/UsersGuide/source/Acknowledgement.rst b/docs/UsersGuide/source/Acknowledgement.rst deleted file mode 100644 index f8847c31d2..0000000000 --- a/docs/UsersGuide/source/Acknowledgement.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _acknowledgement: -.. include:: prolog.inc - ---------------- -Acknowledgement ---------------- - -This user's guide is constructed with contributions from distributed developers. We give our special acknowledgement to these contributors and reviewers, including, but not limited to: - -National Centers for Environmental Prediction (NCEP) Environmental Modeling Center (EMC): |br| -*Jim Abeles, Eric Aligo, Tom Black, Benjamin Blake, Jacob Carley, Dusan Jovic, Jim Purser, Eric Rogers, Ed Strobach, and Fanglin Yang* - -National Center for Atmospheric Research (NCAR) and Developmental Testbed Center (DTC): |br| -*Laurie Carson, Michael Kavulich, Julie Schramm, Don Stark, and Jamie Wolff* - -National Oceanic and Atmospheric Administration (NOAA) Earth System Research Laboratory (ESRL): |br| -*Jeff Beck, Gerard Ketefian, and Linlin Pan* - -The community support and code management effort is sponsored by NOAA's Office of Oceanic and Atmospheric Research (OAR). This work is also facilitated by NCAR. NCAR is supported by the National Science Foundation (NSF). - diff --git a/docs/UsersGuide/source/Chapter1.rst b/docs/UsersGuide/source/Chapter1.rst deleted file mode 100644 index d416db7f68..0000000000 --- a/docs/UsersGuide/source/Chapter1.rst +++ /dev/null @@ -1,27 +0,0 @@ -*************************************** -Summary of the FV3SAR model -*************************************** - -======================== -Basics of the FV3 Solver -======================== - -=========================================== -Handling of Boundary and Initial Conditions -=========================================== - -======================= -The Gnomonic Projection -======================= - ------------------------- -Grid Space Heterogeneity ------------------------- - ------------------------------------------------------------- -Optimization of the Gnomonic Projection for Regional Domains ------------------------------------------------------------- - -================================= -Available Preset Regional Domains -================================= diff --git a/docs/UsersGuide/source/Chapter2.rst b/docs/UsersGuide/source/Chapter2.rst deleted file mode 100644 index cbcd116d43..0000000000 --- a/docs/UsersGuide/source/Chapter2.rst +++ /dev/null @@ -1,264 +0,0 @@ -*************************************** -Software Installation -*************************************** - -=========================================== -System Requirements -=========================================== -The FV3SAR model is supported on the NOAA HPC Theia and NCAR -Supercomputer Cheyenne. Intel is the only currently supported -compiler for building the pre-processing utilities and the FV3SAR model. - -------------------------------------------- -External Libraries -------------------------------------------- -Several external support libraries are required but not included with -the source code. Most of these libraries are installed as part of the -compiler installation. For FV3SAR, these libraries are: - -* Intel compiler -* impi -* ESMFv7.1.0 -* netCDF -* HDF5 -* pnetCDF - -------------------------------------------- -NCEP Libraries -------------------------------------------- -A number of the NCEP (National Center for Environmental Prediction) production -libraries are necessary for building and running the FV3SAR pre-processing utilities -and model (:numref:`Table %s `). These libraries are not part of the source -code distribution. If they are not already installed on your computer platform, you may -have to download some or all the source code from `NCEP `_ -and build the libraries yourself. Note that these libraries must be built with the same compiler -used to build the pre-processing utilities FV3SAR model. Another option is to clone the `git -repository `_ and follow the build instructions. -This currently includes only the libraries used by the FV3SAR model. - -.. _ncep_libs: - -.. table:: *NCEP libraries necessary to build the FV3SAR pre-processing utilities and the model. - X indicates that the library is required.* - - +------------------------+----------------------+-------------+ - | **NCEP Library** | **Pre-Processing** | **FV3SAR** | - +========================+======================+=============+ - | ``bacio/v2.0.1`` | | X | - +------------------------+----------------------+-------------+ - | ``bacio/v2.0.2`` | X | | - +------------------------+----------------------+-------------+ - | ``gfsio/v1.1.0`` | X | | - +------------------------+----------------------+-------------+ - | ``ip/v2.0.0`` | X | X | - +------------------------+----------------------+-------------+ - | ``ip/v3.0.0`` | X | | - +------------------------+----------------------+-------------+ - | ``landsfcutil/v2.1.0`` | X | | - +------------------------+----------------------+-------------+ - | ``nemsio/v2.2.3`` | X | X | - +------------------------+----------------------+-------------+ - | ``nemsiogfs/v2.0.1`` | X | | - +------------------------+----------------------+-------------+ - | ``sfcio/v1.0.0`` | X | | - +------------------------+----------------------+-------------+ - | ``sigio/v2.0.1`` | X | | - +------------------------+----------------------+-------------+ - | ``sp/v2.0.2`` | X | X | - +------------------------+----------------------+-------------+ - | ``w3emc/v2.0.5`` | X | X | - +------------------------+----------------------+-------------+ - | ``w3emc/v2.2.0`` | X | | - +------------------------+----------------------+-------------+ - | ``w3nco/v2.0.6`` | X | X | - +------------------------+----------------------+-------------+ - -.. _ObtainingCode: - -================================ -Obtaining the FV3SAR Source Code -================================ -In order to run FV3SAR, the user must get the FV3SAR workflow scripts -and the NEMSfv3gfs model source code. These steps assume that the -necessary NCEP libraries are built and available as modules on your machine. - -The source code for the FV3SAR workflow, which includes pre-processing utilities, -and the regional model reside in two separate NOAA VLAB repositories. -You will need a NOAA account to check out the code; the ``${USER}`` -variable used below is your NEMS User ID used to log into VLab, -not to be confused with the NEMS (NOAA Environmental Modeling System) -infrastructure. The pre-processing and workflow utilities are located in the -fv3gfs_workflow repository. To clone this repository, create a -directory called ``${BASEDIR}``, clone the repository and check out the -``community`` branch: - -.. code-block:: console - - % mkdir ${BASEDIR} - % cd ${BASEDIR} - % git clone ssh://${USER}@vlab.ncep.noaa.gov:29418/regional_workflow - % cd regional_workflow - % git checkout community - -The model source code is located in the NEMSfv3gfs repository, which includes -three submodules: FMS, FV3, and NEMS. All four of these repositories are hosted -in VLab. You will need to clone the main repository, -checkout the ``regional_fv3_nemsfv3gfs`` branch, and check out the appropriate -branch for each submodule. - -Normally, the appropriate branches to check out are shown in :numref:`Table %s `: - -.. _repo_branches: - -.. table:: *Branches to check out to obtain NEMSfv3gfs source code and three submodules.* - - +-----------------+-----------------------------+ - | **Repository** | **Branch Name** | - +=================+=============================+ - | ``NEMSfv3gfs`` | ``regional_fv3_nemsfv3gfs`` | - +-----------------+-----------------------------+ - | ``FV3`` | ``regional_fv3`` | - +-----------------+-----------------------------+ - | ``FMS`` | ``GFS-FMS`` | - +-----------------+-----------------------------+ - | ``NEMS`` | ``master`` | - +-----------------+-----------------------------+ - -using the following commands: - -.. code-block:: console - - % cd ${BASEDIR} - % git clone --recursive ssh://${USER}@vlab.ncep.noaa.gov:29418/NEMSfv3gfs - % cd NEMSfv3gfs - % git checkout regional_fv3_nemsfv3gfs - % cd FV3 - % git checkout regional_fv3 - % cd ${BASEDIR}/NEMSfv3gfs - % cd FMS - % git checkout GFS-FMS - % cd ${BASEDIR}/NEMSfv3gfs - % cd NEMS - % git checkout master - -However, it turns out that as of 12/13/2018, the code(s) in the “HEAD”s of one or -more of these branches cause(s) one of the post-processing (UPP) tasks in the workflow -for the test run on the RAP domain to hang/fail (the one for forecast hour 6). Thus, -we will for now check out specific commits in these repos that we know will give a -successful run. The hash numbers of these commits are shown in :numref:`Table %s `: - -.. _commit_hashes: - -.. table:: *Specific commits to checkout to achieve a successful run.* - - +-----------------+------------------+ - | **Repository** | **Commit Hash** | - +=================+==================+ - | ``NEMSfv3gfs`` | ``8c97373`` | - +-----------------+------------------+ - | ``FV3`` | ``3ef9be7`` | - +-----------------+------------------+ - | ``FMS`` | ``d4937c8`` | - +-----------------+------------------+ - | ``NEMS`` | ``10325d4`` | - +-----------------+------------------+ - -For convenience, a script named ``checkout_NEMSfv3gfs.sh`` has been created in -the directory ``$BASEDIR/regional_workflow/ush`` to perform these clone and checkout -steps. This script can check out either the heads of the above branches or the -specific commits listed above (the commit hashes are hard-coded into the script). -To have it check out the commits, call this script as follows: - -.. code-block:: console - - % cd ${BASEDIR}/regional_workflow/ush - % ./checkout_NEMSfv3gfs.sh "hash" - -To have the script check out the branch heads, change the first argument from -``hash`` to ``head``, or simply call the script without an argument. - -=========================================== -Building the FV3SAR Source Code -=========================================== -To run the end-to-end FV3SAR forecasting system, the pre-processing utilities, -the FV3SAR model, and the post-porcessing components must be built. This section -describes the steps for the supported compilers on the available platforms. The -directory ``${BASEDIR}`` is assumed to be where the code has been checked -out as described in Section ObtainingCode_. - -To build the FV3SAR pre-processing utilities on theia: - -.. code-block:: console - - % cd ${BASEDIR}/regional_workflow/regional - % ./build_regional theia >& out.build_regional - -Other supported build platforms are ``"cheyenne``, ``wcoss_cray``, or ``odin``. -When the build completes, there should be 9 executables under ``${BASEDIR}/regional_workflow/exec``: - -.. code-block:: console - - % ls ${BASEDIR}/regional_workflow/exec - filter_topo global_chgres make_solo_mosaic - fregrid make_hgrid ml01rg2.x - fregrid_parallel make_hgrid_parallel shave.x - -To build the FV3 model executable: - -.. code-block:: console - - % cd ${BASEDIR}/NEMSfv3gfs/tests - % ./compile.sh ${BASEDIR}/NEMSfv3gfs/FV3 theia.intel "32BIT=Y" 32bit YES NO >& make.out.32bit - -Note the following: - -* The second argument to the ``compile.sh`` script is ``theia.intel``, not just ``theia``. - Other build targets are: - ``cheyenne.gnu, cheyenne.intel, cheyenne.pgi, odin, theia.gnu, theia.intel, theia.pgi`` or ``wcoss_cray``. - -* This is a production build, not a debug build. We don't do the debug build because it is very slow - to run, and you'll time out in the queue. - -* The build takes about 12 minutes to complete. If successful, there should be a file named - ``fv3_32bit.exe`` in the directory ``${BASEDIR}/NEMSfv3gfs/tests``. - -There are other command-line options available when running ``compile.sh`` and are shown in -:numref:`Table %s `. - -.. _build_options: - -.. table:: *Command-line options to build FV3SAR.* - - +---------------------+-----------------------+---------------------------------+--------------+ - | **Argument Number** | **Argument Name** | **Example** | **Optional** | - +=====================+=======================+=================================+==============+ - | 1 | path to FV3 directory | ``${PWD}../FV3`` | No | - +---------------------+-----------------------+---------------------------------+--------------+ - | 2 | ``BUILD_TARGET`` | ``theia.intel, cheyenne.intel`` | No | - +---------------------+-----------------------+---------------------------------+--------------+ - | 3 | ``MAKE_OPT`` | ``DEBUG=Y 32BIT=Y REPRO=N`` | Yes | - +---------------------+-----------------------+---------------------------------+--------------+ - | 4 | ``BUILD_NAME`` | ``32bit`` | Yes | - +---------------------+-----------------------+---------------------------------+--------------+ - | 5 | ``clean_before`` | ``YES`` | Yes | - +---------------------+-----------------------+---------------------------------+--------------+ - | 6 | ``clean_after`` | ``YES`` | Yes | - +---------------------+-----------------------+---------------------------------+--------------+ - -To build with debugging flags, add the following quantities to the ``compile.sh`` command: - -.. code-block:: console - - % ./compile.sh ../FV3 ${BUILD_TARGET} "32BIT=Y DEBUG=Y" 32bit [clean_before] [clean_after] >& make.out.32bit - -The last two optional arguments ``clean_before`` and ``clean_after`` control whether or not to run -make clean to remove temporary files. The default values are ``YES``. Specifying ``NO`` will skip -cleaning step, which will speed up repeating compilation, which is useful for debugging. - -Currently all the fixed fields necessary to run a uniform global case without a nest are in subdirectories -on each supported machine: - -* ``/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_fv3`` on the cray -* ``/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix/fix_fv3`` on theia -* ``/glade/p/ral/jntp/GMTB/FV3GFS_V1_RELEASE/fix/fix_am/`` on Cheyenne - diff --git a/docs/UsersGuide/source/Chapter3.rst b/docs/UsersGuide/source/Chapter3.rst deleted file mode 100644 index 46f16c7922..0000000000 --- a/docs/UsersGuide/source/Chapter3.rst +++ /dev/null @@ -1,170 +0,0 @@ -.. _config_regional_workflow: - -*************************************** -Configuring the FV3SAR Workflow -*************************************** - -The following steps describe how to create a user-specific configuration -file in order to run your experiment in a given environment. - -1. Create a user-specific configuration file named ``config.sh`` in the subdirectory - ``ush`` under the ``$BASEDIR/regional_workflow`` directory containing appropriate - variable settings for your environment, experiment, etc. - - A configuration file named ``config_defaults.sh`` containing default values already - exists in the ``ush`` subdirectory (this file is part of the ``regional_workflow`` - repository). The setup script (called ``setup.sh`` and located in ``ush``) that - will be called in the workflow generation step below first sources ``config_defaults.sh`` - and then sources ``config.sh`` if the latter exists. Thus, any settings in ``config.sh`` - will overwrite the ones in ``config_defaults.sh``. - - Instead of creating a ``config.sh`` script with custom settings, a user can directly - modify the variable settings in ``config_defaults.sh``. However, we do not recommend - this approach because if this file is then pushed to the remote ``regional_workflow`` - repository, that repository will contain in its copy of ``config_defaults.sh`` variable - settings that are specific to your environment and/or experiment. If others then pull - from this repository, they will inherit these settings, but these settings will likely - not be appropriate for their environment and/or experiment. To avoid this situation, - we’ve designed the setup script to read in the local configuration file ``config.sh`` - that users can (and should) create locally in the ush directory and which should remain - out of the repository. We use ``config_defaults.sh`` to document the variables, setting - them to dummy/placeholder values that will be overwritten by the settings in ``config.sh``. - - For the experiment(s) we will run, we do not need to include in ``config.sh`` all the - variables defined in ``config_defaults.sh``; we only need to include the most relevant - ones (the rest will keep their settings in ``config_default.sh``). Thus, include in your - ``config.sh`` the following variable settings: - -.. code-block:: console - - # - MACHINE="THEIA" - ACCOUNT="gsd-fv3" - QUEUE_DEFAULT="debug" - QUEUE_HPSS="service" - QUEUE_RUN_FV3SAR="batch" - # - BASEDIR="/path/to/directory/of/regional_workflow/and/NEMSfv3gfs/clones" - TMPDIR="/path/to/temporary/work/directories" - UPPDIR="/path/to/UPP/executable/directory" - CCPP="false" - # - CDATE="2018060400" - # - fcst_len_hrs="6" - BC_update_intvl_hrs="6" - # - run_title="my_test" - # - #predef_domain="RAP" - predef_domain="HRRR" - # - if [ "$predef_domain" = "RAP" ]; then - layout_x="14" # One possibility: 14 for RAP, 20 for HRRR. - layout_y="14" # One possibility: 14 for RAP, 20 for HRRR. - write_tasks_per_group="14" # One possibility: 14 for RAP, 20 for HRRR. - elif [ "$predef_domain" = "HRRR" ]; then - layout_x="20" # One possibility: 14 for RAP, 20 for HRRR. - layout_y="20" # One possibility: 14 for RAP, 20 for HRRR. - write_tasks_per_group="20" # One possibility: 14 for RAP, 20 for HRRR. - fi - # - preexisting_dir_method="delete" - #preexisting_dir_method="rename" - #preexisting_dir_method="quit" - -Note that the variable values in this code snippet still need to be customized for -your environment and/or experiment. This customization is described in the next step. - -.. warning:: - The following table and the queue information in Step 2 are specific to the NOAA HPC machine theia. - -.. _tasks_queues: - -.. table:: *Workflow task names and the queue submission.* - - +-------------------------+-----------------------------+ - | **Task Name** | **Queue** | - +=========================+=============================+ - | ``make_grid_orog`` | ``QUEUE_DEFAULT`` | - +-------------------------+-----------------------------+ - | ``get_GFS_files`` | ``QUEUE_HPSS`` | - +-------------------------+-----------------------------+ - | ``make_surf_IC_BC0`` | ``QUEUE_DEFAULT`` | - +-------------------------+-----------------------------+ - | ``make_BC1_to_BCend`` | ``QUEUE_DEFAULT`` | - +-------------------------+-----------------------------+ - | ``stage`` | ``QUEUE_DEFAULT`` | - +-------------------------+-----------------------------+ - | ``run_FV3SAR`` | ``QUEUE_RUN_FV3SAR`` | - +-------------------------+-----------------------------+ - | ``post_00 ... post_NN`` | ``QUEUE_DEFAULT`` | - +-------------------------+-----------------------------+ - - -2. Customize the variables in your ``config.sh`` as follows (note that you can find - documentation on what each of these variables represent in ``config_defaults.sh``): - - * Since we're running on theia, leave ``MACHINE`` set to ``"THEIA"``. - - * If you have access to the ``gsd-fv3`` account, leave ``ACCOUNT`` set to ``"gsd-fv3"``. - Otherwise, set ``ACCOUNT`` to one of the accounts you're a member of. - - * Leave ``QUEUE_HPSS`` and ``QUEUE_RUN_FV3SAR`` unchanged, but you can change - ``QUEUE_DEFAULT`` if you like. The workflow tasks and the queues they will - get submitted to are shown in :numref:`Table %s `. - - From :numref:`Table %s `, we can see that the task that gets the - GFS analysis and forecast files (``get_GFS_files``) is submitted to the queue - defined by ``QUEUE_HPSS``, the forecast task (``run_FV3SAR``) is submitted to - the queue defined by ``QUEUE_RUN_FV3SAR``, and all remaining tasks are submitted - to the queue defined by ``QUEUE_DEFAULT``. The theia admins require that any jobs - that access the HPSS run in the ``"service"`` queue, so you have to leave ``QUEUE_HPSS`` - set to ``"service"``. Also, leave ``QUEUE_RUN_FV3SAR`` set to ``"batch"`` because if - you change it to ``“debug”``, the forecast will not complete within the 30 min maximum - walltime of the ``"debug"`` queue, causing the job to time out. For debugging tests, - set ``QUEUE_DEFAULT`` to ``"debug"`` to get the remaining tasks (i.e. the ones other - than ``get_GFS_file`` and ``run_FV3SAR``) in the queue faster. For production runs, - set it to ``"batch"``. - - * Edit ``BASEDIR`` to be your top-level directory. - - * Edit ``TMPDIR`` to be your work directory, wherever you want that to be. A subdirectory - will be created under this directory for each experiment for which you generate a - workflow (using ``generate_FV3SAR_wflow.sh``; see below). Since the run directory that - the setup script (``setup.sh`` in ``ush``) will create will be in ``$BASEDIR/run_dirs``, - it is convenient to set ``TMPDIR`` to ``$BASEDIR/work_dirs`` so that the work and run - directories will be at the same directory level. This is not necessary, however; you - can use another location for ``TMPDIR`` that is independent of ``BASEDIR``. - - * Leave ``UPPDIR`` unchanged for now. This is set to where Jeff's version of UPP is located. - We need to decide where in the SAR directory structure to put UPP and document how to build - it to work with the FV3SAR. - - * Leave ``CDATE``, ``fcst_len_hrs``, and ``BC_update_intvl_hrs`` unchanged since we know - the workflow should complete with these settings. We can explore other values of these - variables later. - - * Change ``run_title`` to a descriptive string for your run. This will get appended to - the end of the names of your run and work directories. - - * Leave ``predef_domain`` set to ``"HRRR"``. If that works, you can later try the - ``"RAP"`` setting. This variable sets the regional domain to one of these two - predefined domains. It does this by setting the grid parameters (i.e. ``RES``, - ``lon_ctr_T6``, etc defined in ``config_defaults.sh``) to predefined values. - If you want a domain other than one of these two defaults, then you have to first - set predef_domain to an empty string and then copy the grid parameters from - ``config_defaults.sh`` to ``config.sh`` and set them to the values you want. - But don't do that for now. - - * Leave the ``layout_x``, ``layout_y``, and write_tasks_per_group settings - unchanged for now. You will likely have to change these if you decide to use a custom domain. - - * Set ``preexisting_dir_method`` to the method you want to use to handle pre-existing - versions of run and work directories. The workflow generation script (``generate_FV3SAR_wflow.sh``, - discussed below) will create run and work directories for your experiment. If one or both - of those directories already exist, the setting of ``preexisting_dir_method`` determines what - will be done with them. If this variable is set to ``"delete"``, pre-existing directories will - be deleted and replaced with new ones; if it is set to ``"rename"``, they will get renamed - (by appending ``"_old001"``, ``"_old002"``, etc to their names); and if it is set to ``"quit"``, - the workflow generation script will fail if it finds any pre-existing directories. diff --git a/docs/UsersGuide/source/Chapter4.rst b/docs/UsersGuide/source/Chapter4.rst deleted file mode 100644 index d3ce887c5a..0000000000 --- a/docs/UsersGuide/source/Chapter4.rst +++ /dev/null @@ -1,261 +0,0 @@ -.. _running_regional_workflow: - -**************************** -Running the FV3SAR Workflow -**************************** -The steps described in this section assume the config.sh file -has been created following the steps outlined in -section :ref:`config_regional_workflow`. In this section, the run and -work directories will be created and a script to run the rocoto -workflow will be generated to run the FV3SAR. - -1. Create the run and work directories and a rocoto workflow XML - for your experiment by running ``generate_FV3SAR_wflow.sh`` in - the ``fv3sar_wflow/ush directory``: - -.. code-block:: console - - % cd $BASEDIR/regional_workflow/ush - % generate_FV3SAR_wflow.sh - -This will create a run directory and a work directory, and it will -put the rocoto workflow XML (called ``FV3SAR_wflow.xml``) in the -run directory. Towards the end of the output that ``generate_FV3SAR_wflow.sh`` -generates, you'll find the locations of the run and work directories -(``RUNDIR=...`` and ``WORKDIR=...``). Note that the workflow always -creates the run directory as a subdirectory under ``$BASEDIR/run_dirs``, -and it creates the work directory as a subdirectory under ``$TMPDIR``. - -The output from ``generate_FV3SAR_wflow.sh`` will contain the commands -you can use to launch the workflow and check on its status (also discussed -in the next step below). It will also contain the line you need to add -to your crontab (using the command ``"crontab -e"``) in order to continuously -resubmit the workflow to the queue. - -2. Regardless of whether or not you added an entry to your crontab for automatic - resubmission, you can go to the run directory and launch the rocoto workflow - as follows (assuming you've already loaded rocoto using ``"module load rocoto"``): - -.. code-block:: console - - % cd $RUNDIR - % rocotorun -w FV3SAR_wflow.xml -d FV3SAR_wflow.db -v 10 - -This will launch the first task in the workflow if it's the first time you're -issuing it. If it's not the first time, it will launch the next task if it -has completed the current one, relaunch the current one if it failed (and -if the number of times you’ve tried to launch the task hasn’t yet exceeded -the maxtries value defined for the task in the workflow XML), or do nothing -if the current one is still running. After issuing this command, you can -check on the status of the workflow using: - -.. code-block:: console - - % rocotostat -w FV3SAR_wflow.xml -d FV3SAR_wflow.db -v 10 - -This will generate a table similar to the following: - -.. code-block:: console - - CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION - ================================================================================================== - 201806040000 make_grid_orog druby://tfe05:33377 SUBMITTING - 0 0.0 - 201806040000 get_GFS_files - - - - - - 201806040000 make_surf_IC_BC0 - - - - - - 201806040000 make_BC1_to_BCend - - - - - - 201806040000 stage - - - - - - 201806040000 run_FV3SAR - - - - - - 201806040000 post_00 - - - - - - 201806040000 post_01 - - - - - - 201806040000 post_02 - - - - - - 201806040000 post_03 - - - - - - 201806040000 post_04 - - - - - - 201806040000 post_05 - - - - - - 201806040000 post_06 - - - - - - -Note that you first have to issue the ``rocotorun`` command above in order to get -updated results from the ``rocotostat`` command. Since it's inconvenient to have -to keep issuing the ``rocotorun`` command to keep the workflow moving along -(i.e. in order to launch the next task if the current one has completed, etc), -you can modify your crontab as described at the end of the output from -``generate_FV3SAR_wflow.sh`` in order to automatically relaunch the workflow with -some specified frequency (e.g. every 3 minutes). - -If all goes well, the workflow should step through all the tasks and finish. The -workflow will have completed when the ``rocotostat`` command generates a table in -which the ``STATE`` for all tasks is ``SUCCEEDED``, something like this: - -.. code-block:: console - - CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION - =============================================================================================== - 201806040000 make_grid_orog 36437677 SUCCEEDED 0 1 262.0 - 201806040000 get_GFS_files 36437726 SUCCEEDED 0 1 71.0 - 201806040000 make_surf_IC_BC0 36437730 SUCCEEDED 0 1 293.0 - 201806040000 make_BC1_to_BCend 36437731 SUCCEEDED 0 1 242.0 - 201806040000 stage 36437765 SUCCEEDED 0 1 38.0 - 201806040000 run_FV3SAR 36437802 SUCCEEDED 0 1 1653.0 - 201806040000 post_00 36438577 SUCCEEDED 0 1 309.0 - 201806040000 post_01 36438624 SUCCEEDED 0 1 294.0 - 201806040000 post_02 36438696 SUCCEEDED 0 1 319.0 - 201806040000 post_03 36438705 SUCCEEDED 0 1 288.0 - 201806040000 post_04 36438441 SUCCEEDED 0 2 293.0 - 201806040000 post_05 36438808 SUCCEEDED 0 1 313.0 - 201806040000 post_06 36438809 SUCCEEDED 0 1 304.0 - -If a job fails, you can find the log file for the job under the run directory -in ``$RUNDIR/log``. Each job will have its own log. This directory will also -have the overall log file for the workflow called ``FV3_$CDATE.log``, where -``CDATE`` is the starting date of the forecast that you set in ``config.sh``. - -================================ -Comparing run output to baseline -================================ - -.. warning:: - The this section is specific to the NOAA HPC machine theia. - -Baseline runs have been created for both the RAP and HRRR domains. The run -directories for these are at the following locations: - -RAP: -``/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_baseline_runs/run_dirs/C384_S0p63_RR3_RAP`` - -HRRR: -``/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_baseline_runs/run_dirs/C384_S1p65_RR5_HRRR`` - -These baselines have been generated using code from the specific commits of the -``NEMSfv3gfs`` repo and its submodules (``FV3``, ``FMS``, and ``NEMS``) mentioned -above (i.e. the ones for which all tasks in the workflows for both the RAP and -HRRR domains complete successfully). Thus, your runs should match these baselines -(assuming you passed the argument ``"hash"`` to the checkout script ``checkout_NEMSfv3gfs.sh`` -described above). As described next, you can use the script ``cmp_rundirs_ncfiles.sh`` -to compare your runs to the baselines. - -The script ``cmp_rundirs_ncfiles.sh`` in the ``ush`` directory compares the NetCDF -files in two specified run directories and their ``INPUT`` subdirectories. Run it as follows: - -.. code-block:: console - - % cd $BASEDIR/regional_workflow/ush - % ./cmp_rundirs_ncfiles.sh "$rundir1" "$rundir2" - -Here, ``rundir1`` and ``rundir2`` are the two run directories you want to compare. -Thus, for example, to compare your RAP run to the baseline, you would use your run -directory for ``rundir1`` and the baseline RAP directory listed above for ``rundir2`` -(or vice versa). - -Below is sample output from running the ``cmp_rundirs_ncfiles.sh`` script to compare -a RAP run to the baseline. You can see that all NetCDF files in the INPUT subdirectory -and the main run directory are identical to their counterparts in the baseline except -for the file ``C384_mosaic.nc``. This is normal because this file contains variables -that contain the full paths to grid files in the run or work directories, and those -full paths will be different because the two run or work directories are different. - -.. code-block:: console - - % ./cmp_rundirs_ncfiles.sh /scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test /scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_baseline_runs/run_dirs/C384_S0p63_RR3_RAP - - rundir1 = "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test" - rundir2 = "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_baseline_runs/run_dirs/C384_S0p63_RR3_RAP" - - Comparing files in subdirectory "INPUT" ... - =========================================== - - Comparing file "C384_grid.tile7.halo3.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing file "C384_grid.tile7.halo4.nc" in subdirectory "INPUT" ... - Files are identical. - - File "C384_grid.tile7.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - Comparing file "C384_mosaic.nc" in subdirectory "INPUT" ... - DIFFER : VARIABLE : gridlocation : POSITION : [34] : VALUES : F <> r - ===>>> FILES ARE DIFFERENT!!! - - Comparing file "C384_oro_data.tile7.halo0.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing file "C384_oro_data.tile7.halo4.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing file "gfs_bndy.tile7.000.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing file "gfs_bndy.tile7.006.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing file "gfs_ctrl.nc" in subdirectory "INPUT" ... - Files are identical. - - File "gfs_data.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - Comparing file "gfs_data.tile7.nc" in subdirectory "INPUT" ... - Files are identical. - - File "grid_spec.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - File "grid.tile7.halo4.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - File "oro_data.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - File "oro_data.tile7.halo4.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - File "sfc_data.nc" in "/scratch3/BMC/fim/Gerard.Ketefian/FV3SAR_test_gsk/run_dirs/C384_S0p63_RR3_RAP_my_test/INPUT" is a symbolic link. Skipping. - - - Comparing file "sfc_data.tile7.nc" in subdirectory "INPUT" ... - Files are identical. - - Comparing files in subdirectory "." ... - ======================================= - - Comparing file "atmos_static.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf000.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf001.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf002.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf003.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf004.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf005.nc" in subdirectory "." ... - Files are identical. - - Comparing file "dynf006.nc" in subdirectory "." ... - Files are identical. - - Comparing file "grid_spec.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf000.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf001.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf002.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf003.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf004.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf005.nc" in subdirectory "." ... - Files are identical. - - Comparing file "phyf006.nc" in subdirectory "." ... - Files are identical. - diff --git a/docs/UsersGuide/source/Introduction.rst b/docs/UsersGuide/source/Introduction.rst deleted file mode 100644 index 3b304d9e96..0000000000 --- a/docs/UsersGuide/source/Introduction.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _introduction: -.. include:: prolog.inc - ------------- -Introduction ------------- - -This document contains the Finite-Volume Cubed-Sphere Standalone Regional (FV3SAR) Model User's Guide. It provides an overview of the FV3SAR, the fundamentals of obtaining, building and running the FV3SAR. There is also a description of the FV3SAR community workflow and its components. - -For the latest version of the released code, please visit the `FV3SAR DTC Website `_ - -Please send questions and comments to the help desk: `xxx-help@ucar.edu` - -This document and the annual releases are made available through a community effort jointly led by the Developmental Testbed Center (DTC) and the National Centers for Environmental Prediction (NCEP) Environmental Modeling Center (EMC), in collaboration with other developers. To help sustain this effort, we recommend for those who use the community release, the helpdesk, the User's Guide, and other DTC services, please refer to this community effort in their work and publications. - - -How To Use This Document ------------------------- - -This table describes the type changes and symbols used in this guide. - -+------------------------+------------------------------+---------------------------------------+ -| **Typeface or Symbol** | **Meaning** | **Example** | -+========================+==============================+=======================================+ -| ``AaBbCc123`` | The names of commands, | Edit your ``.bashrc`` |br| | -| | files, and directories; |br| | Use ``ls -a`` to list all files. |br| | -| | on-screen computer output | ``host$ You have mail!`` | -+------------------------+------------------------------+---------------------------------------+ -| :mod:`AaBbCc123` | What you type contrasted | ``host$`` :mod:`su` | -| | with on-screen computer | | -| | output | | -+------------------------+------------------------------+---------------------------------------+ -| ``%`` | Command-line prompt | ``% cd $TOP_DIR`` | -+------------------------+------------------------------+---------------------------------------+ - -Following these typefaces and conventions, shell commands, code examples, namelist varialbes, etc. -will be presented in this style: - -.. code-block:: console - - % mkdir ${TOP_DIR} diff --git a/docs/UsersGuide/source/_static/custom.css b/docs/UsersGuide/source/_static/custom.css deleted file mode 100644 index 25f207a223..0000000000 --- a/docs/UsersGuide/source/_static/custom.css +++ /dev/null @@ -1,6 +0,0 @@ -.red { color: red; } -.bolditalic { - font-family: "Courier New", Courier, monospace; - font-weight: bold; - font-style: italic; -} diff --git a/docs/UsersGuide/source/_templates/.gitignore b/docs/UsersGuide/source/_templates/.gitignore deleted file mode 100644 index 5e7d2734cf..0000000000 --- a/docs/UsersGuide/source/_templates/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/docs/UsersGuide/source/conf.py b/docs/UsersGuide/source/conf.py deleted file mode 100644 index b957507406..0000000000 --- a/docs/UsersGuide/source/conf.py +++ /dev/null @@ -1,202 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Configuration file for the Sphinx documentation builder. -# -# This file does only contain a selection of the most common options. For a -# full list see the documentation: -# http://www.sphinx-doc.org/en/master/config - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('.')) - - -# -- Project information ----------------------------------------------------- - -project = 'FV3SAR' -copyright = '2019, ' -author = ' ' - -# The short X.Y version -version = '' -# The full version, including alpha/beta/rc tags -release = 'v1.0' - -numfig = True - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', - 'sphinxcontrib.bibtex' -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The master toctree document. -master_doc = 'index' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'classic' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} -html_theme_options = {"body_max_width": "none"} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -def setup(app): - app.add_stylesheet('custom.css') # may also be an URL - -# Custom sidebar templates, must be a dictionary that maps document names -# to template names. -# -# The default sidebars (for documents that don't match any pattern) are -# defined by theme itself. Builtin themes are using these templates by -# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', -# 'searchbox.html']``. -# -# html_sidebars = {} - - -# -- Options for HTMLHelp output --------------------------------------------- - -# Output file base name for HTML help builder. -htmlhelp_basename = 'FV3SAR' - - -# -- Options for LaTeX output ------------------------------------------------ - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - 'pointsize': '11pt', - - # Additional stuff for the LaTeX preamble. - 'preamble': r''' - \usepackage{charter} - \usepackage[defaultsans]{lato} - \usepackage{inconsolata} - ''', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'FV3SAR.tex', 'FV3SAR Documentation', - ' ', 'manual'), -] - - -# -- Options for manual page output ------------------------------------------ - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'FV3SAR', 'FV3SAR Documentation', - [author], 1) -] - - -# -- Options for Texinfo output ---------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'FV3SAR', 'FV3SAR Documentation', - author, 'FV3SAR', 'One line description of project.', - 'Miscellaneous'), -] - - -# -- Options for Epub output ------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = project - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# -# epub_identifier = '' - -# A unique identification for the text. -# -# epub_uid = '' - -# A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] - - -# -- Extension configuration ------------------------------------------------- - -# -- Options for intersphinx extension --------------------------------------- - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/': None} - -# -- Options for todo extension ---------------------------------------------- - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True diff --git a/docs/UsersGuide/source/index.rst b/docs/UsersGuide/source/index.rst deleted file mode 100644 index 1a0a50a932..0000000000 --- a/docs/UsersGuide/source/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. ccpp_docs documentation master file, created by - sphinx-quickstart on Tue Feb 12 08:48:32 2019. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -FV3SAR User's Guide -=================== - -.. toctree:: - :maxdepth: 3 - - Introduction - Chapter1 - Chapter2 - Chapter3 - Chapter4 - Acknowledgement diff --git a/docs/UsersGuide/source/prolog.inc b/docs/UsersGuide/source/prolog.inc deleted file mode 100644 index 516e6eb6ae..0000000000 --- a/docs/UsersGuide/source/prolog.inc +++ /dev/null @@ -1,4 +0,0 @@ -.. # define a hard line break for HTML -.. |br| raw:: html - -
diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_ics.local b/modulefiles/tasks/wcoss_cray/get_extrn_ics.local deleted file mode 100644 index d540aea2cc..0000000000 --- a/modulefiles/tasks/wcoss_cray/get_extrn_ics.local +++ /dev/null @@ -1,11 +0,0 @@ -#%Module##################################################### -## Module file for get_extrn_ics task. -############################################################# - -module purge - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local deleted file mode 100644 index d7ec3794e4..0000000000 --- a/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local +++ /dev/null @@ -1,11 +0,0 @@ -#%Module##################################################### -## Module file for get_extrn_lbcs task. -############################################################# - -module purge - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_cray/get_obs.local b/modulefiles/tasks/wcoss_cray/get_obs.local deleted file mode 100644 index 43ce12ebbc..0000000000 --- a/modulefiles/tasks/wcoss_cray/get_obs.local +++ /dev/null @@ -1,9 +0,0 @@ -#%Module##################################################### -## Module file for get_obs_ccpa, get_obs_mrms, get_obs_ndas -############################################################# - -module purge - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - diff --git a/modulefiles/tasks/wcoss_cray/make_grid.local b/modulefiles/tasks/wcoss_cray/make_grid.local deleted file mode 100644 index edee5d7249..0000000000 --- a/modulefiles/tasks/wcoss_cray/make_grid.local +++ /dev/null @@ -1,15 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 - -module list diff --git a/modulefiles/tasks/wcoss_cray/make_ics.local b/modulefiles/tasks/wcoss_cray/make_ics.local deleted file mode 100644 index 04eb686e14..0000000000 --- a/modulefiles/tasks/wcoss_cray/make_ics.local +++ /dev/null @@ -1,17 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 - diff --git a/modulefiles/tasks/wcoss_cray/make_lbcs.local b/modulefiles/tasks/wcoss_cray/make_lbcs.local deleted file mode 100644 index 04eb686e14..0000000000 --- a/modulefiles/tasks/wcoss_cray/make_lbcs.local +++ /dev/null @@ -1,17 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 - diff --git a/modulefiles/tasks/wcoss_cray/make_orog.local b/modulefiles/tasks/wcoss_cray/make_orog.local deleted file mode 100644 index c431755327..0000000000 --- a/modulefiles/tasks/wcoss_cray/make_orog.local +++ /dev/null @@ -1,12 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module list diff --git a/modulefiles/tasks/wcoss_cray/make_sfc_climo.local b/modulefiles/tasks/wcoss_cray/make_sfc_climo.local deleted file mode 100644 index c431755327..0000000000 --- a/modulefiles/tasks/wcoss_cray/make_sfc_climo.local +++ /dev/null @@ -1,12 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module list diff --git a/modulefiles/tasks/wcoss_cray/run_fcst.local b/modulefiles/tasks/wcoss_cray/run_fcst.local deleted file mode 100644 index edee5d7249..0000000000 --- a/modulefiles/tasks/wcoss_cray/run_fcst.local +++ /dev/null @@ -1,15 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 - -module list diff --git a/modulefiles/tasks/wcoss_cray/run_post.local b/modulefiles/tasks/wcoss_cray/run_post.local deleted file mode 100644 index 4529dd2f37..0000000000 --- a/modulefiles/tasks/wcoss_cray/run_post.local +++ /dev/null @@ -1,18 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /usrx/local/prod/modulefiles -module load hpss/4.1.0.3 - -module use /usrx/local/nceplibs/modulefiles -module load srw-app-python/1.0.0 - - diff --git a/modulefiles/tasks/wcoss_cray/run_vx.local b/modulefiles/tasks/wcoss_cray/run_vx.local deleted file mode 100644 index 6dbf48c0f9..0000000000 --- a/modulefiles/tasks/wcoss_cray/run_vx.local +++ /dev/null @@ -1,13 +0,0 @@ -#%Module - -module use /opt/cray/ari/modulefiles -module load alps -module load gni-headers -module load pmi/5.0.11 -module load rca -module load udreg -module load ugni -module load xpmem - -module use /gpfs/hps3/emc/meso/noscrub/emc.metplus/modulefiles -module load met/10.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local index 1502c04d56..9900015d1f 100644 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local +++ b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local @@ -4,6 +4,3 @@ module purge module load HPSS/5.0.2.5 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local index 74e8f878c1..4a8b7431f9 100644 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local +++ b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local @@ -4,6 +4,3 @@ module purge module load HPSS/5.0.2.5 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_post.local b/modulefiles/tasks/wcoss_dell_p3/run_post.local deleted file mode 100644 index 9e6f62305a..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/run_post.local +++ /dev/null @@ -1,7 +0,0 @@ -#%Module - -module load lsf/10.1 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 - diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 21db7b20a4..7c840a955f 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -558,7 +558,7 @@ resubmit SRW workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - printf "%s" "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" + printf "%s\n" "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" else # Add a newline to the end of crontab_contents only if it is not empty. # This is needed so that when CRONTAB_LINE is printed out, it appears on From a78ffeff93754158fb9a54ef0ca57d0a6736d1f1 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Tue, 15 Mar 2022 14:31:17 -0600 Subject: [PATCH 133/203] Remove old/obsolete suites (CPT_v0, GSD_v0, GSD_SAR) from workflow (#697) ## DOCUMENTATION: This PR removes the `FV3_CPT_v0`, `FV3_GSD_v0`, and `FV3_GSD_SAR` suites from the workflow. This consists of: 1. Removing these suites from ex-scripts, templates, and the set of valid values for the variable `CCPP_PHYS_SUITE`, 2. Removing the `diag_table_...` and `field_table_...` files for these suites. 3. Removing WE2E tests in the `grids_extrn_mdls_suites_community` category (which are tests to make sure that specific combinations of grids, external models, and suites work well together) that use these suites. 4. Modifying the three WE2E tests in the `wflow_features` category (`get_from_HPSS_ics_HRRR_lbcs_RAP`, `get_from_HPSS_ics_RAP_lbcs_RAP`, and `specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE`) that happen to use the `FV3_GSD_SAR` suite such that they now use the `FV3_HRRR` suite. (There are no such tests that use the `FV3_CPT_v0` and `FV3_GSD_v0` suites.) Note that we don't remove these tests because their purpose is not to test the suite but to test fetching of files from HPSS (`get_from_HPSS_ics_HRRR_lbcs_RAP` and `get_from_HPSS_ics_RAP_lbcs_RAP`) and to test that the experiment variables `DT_ATMOS`, `LAYOUT_X`, `LAYOUT_Y`, and `BLOCKSIZE` can be correctly specified in the user's experiment configuration file (`specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE` 5. Updating comments in scripts that may refer to one of these three suites. This PR also makes improvements to the `tests/get_expts_status.sh` script that is used to check the status of a set of experiments in a specified directory. ## DEPENDENCIES: PR #[224](https://github.com/ufs-community/ufs-srweather-app/pull/224) in the `ufs-srweather-app` repo. ## TESTS CONDUCTED: Ran the following tests on Hera: ``` grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR get_from_HPSS_ics_HRRR_lbcs_RAP get_from_HPSS_ics_RAP_lbcs_RAP specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE ``` All succeeded. Also, since the modifications to the `FV3.input.yml` file affect the `FV3_RRFS_v1alpha`, `FV3_RRFS_v1beta`, and `FV3_HRRR` suites, the `input.nml` files for these suites generated using the (original) `develop` branch were compared to the ones generated using this branch/PR, and all were found to be identical. ## ISSUE (optional): Resolves Issue #668. --- scripts/exregional_make_ics.sh | 4 +- scripts/exregional_make_lbcs.sh | 4 +- tests/WE2E/get_expts_status.sh | 38 +- ..._AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh | 50 --- ..._AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh | 25 -- ...S_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh | 25 -- ...US_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 26 -- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh | 25 -- ...5km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh | 25 -- ...S_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh | 25 -- ...US_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 26 -- ...NUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh | 26 -- ...NUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh | 25 -- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 26 -- ...3km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh | 25 -- ...NUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh | 26 -- .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 2 +- .../config.get_from_HPSS_ics_RAP_lbcs_RAP.sh | 2 +- ...ig.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh | 2 +- ush/set_namelist.py | 4 +- ush/templates/FV3.input.yml | 138 +------ ush/templates/diag_table.FV3_CPT_v0 | 311 --------------- ush/templates/diag_table.FV3_GSD_SAR | 362 ------------------ ush/templates/diag_table.FV3_GSD_SAR_v1 | 341 ----------------- ush/templates/diag_table.FV3_GSD_v0 | 362 ------------------ ush/templates/field_table.FV3_CPT_v0 | 67 ---- ush/templates/field_table.FV3_GSD_SAR | 65 ---- ush/templates/field_table.FV3_GSD_SAR_v1 | 65 ---- ush/templates/field_table.FV3_GSD_v0 | 65 ---- ush/valid_param_vals.sh | 3 - 30 files changed, 51 insertions(+), 2139 deletions(-) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh delete mode 100755 ush/templates/diag_table.FV3_CPT_v0 delete mode 100644 ush/templates/diag_table.FV3_GSD_SAR delete mode 100644 ush/templates/diag_table.FV3_GSD_SAR_v1 delete mode 100644 ush/templates/diag_table.FV3_GSD_v0 delete mode 100644 ush/templates/field_table.FV3_CPT_v0 delete mode 100644 ush/templates/field_table.FV3_GSD_SAR delete mode 100644 ush/templates/field_table.FV3_GSD_SAR_v1 delete mode 100644 ush/templates/field_table.FV3_GSD_v0 diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index ca27553977..8a63b19786 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -138,12 +138,10 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GFS_2017_gfdlmp" | \ "FV3_GFS_2017_gfdlmp_regional" | \ "FV3_GFS_v16" | \ - "FV3_GFS_v15p2" | "FV3_CPT_v0" ) + "FV3_GFS_v15p2" ) varmap_file="GFSphys_var_map.txt" ;; # - "FV3_GSD_v0" | \ - "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ "FV3_GFS_v15_thompson_mynn_lam3km" | \ diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 23acdc1f14..e1d40fb88c 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -136,12 +136,10 @@ case "${CCPP_PHYS_SUITE}" in "FV3_GFS_2017_gfdlmp" | \ "FV3_GFS_2017_gfdlmp_regional" | \ "FV3_GFS_v16" | \ - "FV3_GFS_v15p2" | "FV3_CPT_v0" ) + "FV3_GFS_v15p2" ) varmap_file="GFSphys_var_map.txt" ;; # - "FV3_GSD_v0" | \ - "FV3_GSD_SAR" | \ "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ "FV3_GFS_v15_thompson_mynn_lam3km" | \ diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 997bfb6b8f..01b127d9fe 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -309,40 +309,50 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename" #----------------------------------------------------------------------- # launch_wflow_fn="launch_FV3LAM_wflow.sh" +launch_wflow_log_fn="log.launch_FV3LAM_wflow" num_tail_lines="40" for (( i=0; i<=$((num_expts-1)); i++ )); do expt_subdir="${expt_subdirs[$i]}" - print_info_msg "\ + msg="\ $separator -Checking workflow status of experiment: \"${expt_subdir}\"" +Checking workflow status of experiment \"${expt_subdir}\" ..." + print_info_msg "$msg" # # Change location to the experiment subdirectory, call the workflow launch -# script to update the launch log file, and capture the output from that -# call. +# script to update the workflow launch log file, and capture the output +# from that call. # cd_vrfy "${expt_subdir}" launch_msg=$( "${launch_wflow_fn}" 2>&1 ) - log_tail=$( tail -n ${num_tail_lines} log.launch_FV3LAM_wflow ) -# -# Record the tail from the log file into the status report file. -# - print_info_msg "$msg" >> "${expts_status_fp}" - print_info_msg "${log_tail}" >> "${expts_status_fp}" + log_tail=$( tail -n ${num_tail_lines} "${launch_wflow_log_fn}" ) # # Print the workflow status to the screen. # wflow_status=$( printf "${log_tail}" | grep "Workflow status:" ) # wflow_status="${wflow_status## }" # Not sure why this doesn't work to strip leading spaces. - wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) - msg="${wflow_status}" - print_info_msg "$msg" - + wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) # Remove leading spaces. + print_info_msg "${wflow_status}" print_info_msg "\ $separator " # +# Combine message above with the last num_tail_lines lines from the workflow +# launch log file and place the result in the status report file. +# + msg=$msg" +${wflow_status} + +The last ${num_tail_lines} lines of this experiment's workflow launch log file +(\"${launch_wflow_log_fn}\") are: + +${log_tail} + + +" + print_info_msg "$msg" >> "${expts_status_fp}" +# # Change location back to the experiments base directory. # cd_vrfy "${expts_basedir}" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 18ad96810d..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,50 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the GSD_HRRR_AK_50km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the RAP. -# -# Note that this test specifies computational resource parameters for -# the MAKE_ICS_TN, MAKE_LBCS_TN, and RUN_POST_TN rocoto tasks in order -# allow the chgres_cube and UPP codes to complete these tasks successfully -# on this very coarse grid. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="GSD_HRRR_AK_50km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" -# -# For a coarse grid such as this, the number of MPI processes (= NNODES*PPN) -# can't be too large for the make_ics and make_lbcs tasks (both of which -# use chgres_cube); otherwise, the chgres_cube code will fail. -# -NNODES_MAKE_ICS="1" -PPN_MAKE_ICS="12" -WTIME_MAKE_ICS="00:30:00" - -NNODES_MAKE_LBCS="1" -PPN_MAKE_LBCS="12" -WTIME_MAKE_LBCS="00:30:00" -# -# For a coarse grid such as this, the number of MPI processes (= NNODES*PPN) -# can't be too large for the run_post metatask (which uses the UPP code); -# otherwise, the UPP code will fail. -# -NNODES_RUN_POST="1" -PPN_RUN_POST="12" -WTIME_RUN_POST="00:30:00" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 44642ece92..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_AK_13km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_AK_13km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index e2da03a3bb..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_AK_3km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_AK_3km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="RAP" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 18a62f18d5..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_13km grid using the GSD_SAR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_13km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh deleted file mode 100644 index a313c8d1dc..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the FV3GFS. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh deleted file mode 100644 index aae81d6ebf..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the CPT_v0 -# physics suite with ICs and LBCs derived from the GSMGFS. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_CPT_v0" - -EXTRN_MDL_NAME_ICS="GSMGFS" -EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh deleted file mode 100644 index cc239add0c..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the HRRR. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 1bd56f0001..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh deleted file mode 100644 index 21ab6e7abf..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the GSD_v0 -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_v0" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190520" -DATE_LAST_CYCL="20190520" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh deleted file mode 100644 index 8a7c1c102e..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the GSD_SAR -# physics suite with ICs and LBCs derived from the NAM. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="NAM" -EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20150602" -DATE_LAST_CYCL="20150602" -CYCL_HRS=( "12" ) - -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 35301c4666..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_3km grid using the GSD_SAR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh deleted file mode 100644 index 763945bf8f..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_NA_13km grid using the GSD_v0 -# physics suite with ICs and LBCs derived from the FV3GFS. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_NA_13km" -CCPP_PHYS_SUITE="FV3_GSD_v0" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh deleted file mode 100644 index 4c2ff9f192..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_SUBCONUS_3km grid using the GSD_SAR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_SUBCONUS_3km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh index 84e93130f7..d535bc5d54 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh @@ -13,7 +13,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" +CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh index 62fafdf86c..bed8ceb8c8 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh @@ -11,7 +11,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" +CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="RAP" EXTRN_MDL_NAME_LBCS="RAP" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index 68fa315378..18699dd51d 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -11,7 +11,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GSD_SAR" +CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" diff --git a/ush/set_namelist.py b/ush/set_namelist.py index f16a798e7b..43bbeb8a66 100755 --- a/ush/set_namelist.py +++ b/ush/set_namelist.py @@ -40,7 +40,7 @@ To produce a namelist (fv3_expt.nml) by specifying a physics package: - set_namelist.py -n templates/input.nml.FV3 -c templates/FV3.input.yml FV3_GSD_SAR + set_namelist.py -n templates/input.nml.FV3 -c templates/FV3.input.yml FV3_HRRR -o fv3_expt.nml To produce a YAML file (fv3_namelist.yml) from a user namelist: @@ -171,7 +171,7 @@ def parse_args(): ) parser.add_argument('-n', '--basenml', dest='nml', - help='Full path the input Fortran namelist. Optional.', + help='Full path to the input Fortran namelist. Optional.', type=file_exists, ) parser.add_argument('-t', '--type', diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index b971671dd0..d64a580792 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -1,22 +1,23 @@ -# This configuration file maintains all the community-supported sets of -# namelist options. To build a namelist for one of these configurations, -# use the Python helper script ush/set_namelist.py and provide this file -# and the desired section via the -c option. - -# The FV3_GSD_v0 section is empty as it provides the "base namelist" to -# which all the modifications here are provided. Please see +# This configuration file maintains the modifications that need to be +# made to the base FV3 namelist specified in +# # ush/templates/input.nml.FV3 -# for the full set of namelist options that are used for FV3 in that -# configuration. - +# +# to obtain the namelist for each physics suite that the SRW App can +# run with. To build a namelist for one of these configurations, use +# the Python helper script +# +# ush/set_namelist.py +# +# and provide this file and the desired section via the -c option. -FV3_GSD_v0: -FV3_GSD_SAR: - gfs_physics_nml: &gsd_sar_phys +FV3_RRFS_v1alpha: + gfs_physics_nml: &RRFS_v1alpha_phys do_deep: False - imfdeepcnv: 0 - imfshalcnv: 0 + do_mynnsfclay: False + imfdeepcnv: -1 + imfshalcnv: -1 iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -29,26 +30,13 @@ FV3_GSD_SAR: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - -FV3_RRFS_v1alpha: - gfs_physics_nml: - <<: *gsd_sar_phys - do_deep: False - do_mynnsfclay: False - imfdeepcnv: -1 - imfshalcnv: -1 lsm: 2 lsoil_lsm: 4 FV3_RRFS_v1beta: gfs_physics_nml: - <<: *gsd_sar_phys - do_deep: False + <<: *RRFS_v1alpha_phys do_mynnsfclay: True - imfdeepcnv: -1 - imfshalcnv: -1 - lsm: 2 - lsoil_lsm: 4 FV3_HRRR: fv_core_nml: @@ -63,7 +51,7 @@ FV3_HRRR: nord_tr: 2 nrows_blend: 10 gfs_physics_nml: - <<: *gsd_sar_phys + <<: *RRFS_v1alpha_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True do_sfcperts: !!python/none @@ -73,9 +61,9 @@ FV3_HRRR: do_gsl_drag_ls_bl: True iaer: 5111 icliq_sw: 2 - imfdeepcnv: -1 - imfshalcnv: -1 iovr: 3 + lsm: 3 + lsoil_lsm: 9 FV3_GFS_2017_gfdlmp: atmos_model_nml: @@ -394,89 +382,3 @@ FV3_GFS_v16: ldebug: False surf_map_nml: !!python/none -FV3_CPT_v0: - fv_core_nml: - <<: *gfs_v15_fv_core - dnats: 0 - fv_sg_adj: 450 - k_split: 6 - n_split: 6 - n_sponge: 24 - nudge_qv: False - rf_fast: !!python/none - gfdl_cloud_microphysics_nml: - <<: *gfs_gfdl_cloud_mp - mp_time: 150.0 - sedi_transport: True - tau_i2s: 1500.0 - tau_l2v: 225.0 - tau_v2l: 150.0 - gfs_physics_nml: - aero_in: False - bl_mynn_edmf: !!python/none - bl_mynn_edmf_mom: !!python/none - bl_mynn_tkeadvect: !!python/none - ccwf: [1.0, 1.0] - crtrh: [0.93, 0.9, 0.95] - cs_parm: [8.0, 4.0, 1000.0, 3500.0, 20.0, 1.0, 0.0, 1.0, 0.6, 0.0] - cscnv: True - ctei_rm: [0.6, 0.23] - dlqf: [0.25, 0.05] - do_aw: True - do_sb_physics: True - do_mynnedmf: !!python/none - do_mynnsfclay: !!python/none - do_shoc: False - fhcyc: 0.0 - fhlwr: 3600.0 - fhswr: 3600.0 - fprcp: 2 - hybedmf: True - iccn: False - icloud_bl: !!python/none - imp_physics: 10 - imfshalcnv: 2 - imfdeepcnv: -1 - iopt_alb: 2 - iopt_btr: 1 - iopt_crs: 1 - iopt_dveg: 2 - iopt_frz: 1 - iopt_inf: 1 - iopt_rad: 1 - iopt_run: 1 - iopt_sfc: 1 - iopt_snf: 4 - iopt_stc: 1 - iopt_tbot: 2 - lndp_type: !!python/none - lradar: False - lsm: 1 - lsoil: !!python/none - lsoil_lsm: !!python/none - ltaerosol: False - max_lat: 4000 - max_lon: 8000 - mg_dcs: 200.0 - mg_do_graupel: True - mg_do_hail: False - mg_do_ice_gmao: False - mg_do_liq_liu: True - mg_qcvar: 1.0 - mg_ts_auto_ice: [180.0, 900.0] - n_var_lndp: !!python/none - ncld: 2 - pdfflag: 4 - random_clds: True - ras: False - rhcmax: 0.9999999 - shal_cnv: True - shcnvcw: False - shoc_cld: False - shoc_parm: [7000.0, 1.0, 2.0, 0.7, -999.0] - ttendlim: !!python/none - xkzm_h: 0.5 - xkzm_m: 0.5 - xkzm_s: 1.0 - surf_map_nml: !!python/none - diff --git a/ush/templates/diag_table.FV3_CPT_v0 b/ush/templates/diag_table.FV3_CPT_v0 deleted file mode 100755 index c3e906af4a..0000000000 --- a/ush/templates/diag_table.FV3_CPT_v0 +++ /dev/null @@ -1,311 +0,0 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} - -"grid_spec", -1, "months", 1, "days", "time" -"atmos_4xdaily", 6, "hours", 1, "days", "time" -"atmos_static", -1, "hours", 1, "hours", "time" -"fv3_history", 1, "years", 1, "hours", "time" -"fv3_history2d", 1, "years", 1, "hours", "time" - -# -#======================= -# ATMOSPHERE DIAGNOSTICS -#======================= -### -# grid_spec -### - "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, - "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, -### -# 4x daily output -### - "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 -#### - "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 -#### - "dynamics", "h1000", "h1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h850", "h850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h700", "h700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h500", "h500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h200", "h200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h100", "h100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h50", "h50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "h10", "h10", "atmos_4xdaily", "all", .false., "none", 2 -#### -#"dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 -#### - "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 -#### -#"dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 -#"dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 -#"dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 -#"dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 -#"dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 - "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 -### -# gfs static data -### - "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 - "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 - "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 - "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 - "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 -### -# FV3 variabls needed for NGGPS evaluation -### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "water_nc", "spncw", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_nc", "spnci", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rain_nc", "spncr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snow_nc", "spncs", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel_nc", "graupelnc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sgs_tke", "tke", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -# Stochastic physics -"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/diag_table.FV3_GSD_SAR b/ush/templates/diag_table.FV3_GSD_SAR deleted file mode 100644 index 0d2849c346..0000000000 --- a/ush/templates/diag_table.FV3_GSD_SAR +++ /dev/null @@ -1,362 +0,0 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} - -"grid_spec", -1, "months", 1, "days", "time" -"atmos_static", -1, "hours", 1, "hours", "time" -#"atmos_4xdaily", 1, "hours", 1, "days", "time" -"fv3_history", 1, "years", 1, "hours", "time" -"fv3_history2d", 1, "years", 1, "hours", "time" - -# -#======================= -# ATMOSPHERE DIAGNOSTICS -#======================= -### -# grid_spec -### - "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, - "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, -### -# 4x daily output -### -# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 -### -# gfs static data -### - "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 - "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 - "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 - "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 - "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 -### -# FV3 variabls needed for NGGPS evaluation -### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 - -"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 -#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt5", "soilt5" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt6", "soilt6" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt7", "soilt7" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt8", "soilt8" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt9", "soilt9" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw5", "soilw5" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw6", "soilw6" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw7", "soilw7" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw8", "soilw8" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw9", "soilw9" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_5", "soill5", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_6", "soill6", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_7", "soill7", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_8", "soill8", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_9", "soill9", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -# Aerosols (CCN, IN) from Thompson microphysics -"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 -# Cloud effective radii from Thompson and WSM6 microphysics -"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 -# Prognostic/diagnostic variables from MYNN -"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 -# Prognostic/diagnostic variables from RUC LSM -"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -# Stochastic physics -"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 - -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/diag_table.FV3_GSD_SAR_v1 b/ush/templates/diag_table.FV3_GSD_SAR_v1 deleted file mode 100644 index 99b7652910..0000000000 --- a/ush/templates/diag_table.FV3_GSD_SAR_v1 +++ /dev/null @@ -1,341 +0,0 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} - -"grid_spec", -1, "months", 1, "days", "time" -"atmos_static", -1, "hours", 1, "hours", "time" -#"atmos_4xdaily", 1, "hours", 1, "days", "time" -"fv3_history", 1, "years", 1, "hours", "time" -"fv3_history2d", 1, "years", 1, "hours", "time" - -# -#======================= -# ATMOSPHERE DIAGNOSTICS -#======================= -### -# grid_spec -### - "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, - "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, -### -# 4x daily output -### -# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 -### -# gfs static data -### - "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 - "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 - "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 - "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 - "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 -### -# FV3 variabls needed for NGGPS evaluation -### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 - -"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 -#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -# Aerosols (CCN, IN) from Thompson microphysics -"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 -# Cloud effective radii from Thompson and WSM6 microphysics -"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 -# Prognostic/diagnostic variables from MYNN -"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 -# Prognostic/diagnostic variables from RUC LSM -"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 - -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/diag_table.FV3_GSD_v0 b/ush/templates/diag_table.FV3_GSD_v0 deleted file mode 100644 index 0d2849c346..0000000000 --- a/ush/templates/diag_table.FV3_GSD_v0 +++ /dev/null @@ -1,362 +0,0 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} - -"grid_spec", -1, "months", 1, "days", "time" -"atmos_static", -1, "hours", 1, "hours", "time" -#"atmos_4xdaily", 1, "hours", 1, "days", "time" -"fv3_history", 1, "years", 1, "hours", "time" -"fv3_history2d", 1, "years", 1, "hours", "time" - -# -#======================= -# ATMOSPHERE DIAGNOSTICS -#======================= -### -# grid_spec -### - "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, - "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, -### -# 4x daily output -### -# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 -### -# gfs static data -### - "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 - "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 - "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 - "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 - "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 -### -# FV3 variabls needed for NGGPS evaluation -### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 - -"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 -#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt5", "soilt5" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt6", "soilt6" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt7", "soilt7" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt8", "soilt8" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt9", "soilt9" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw5", "soilw5" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw6", "soilw6" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw7", "soilw7" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw8", "soilw8" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw9", "soilw9" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_5", "soill5", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_6", "soill6", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_7", "soill7", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_8", "soill8", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_9", "soill9", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -# Aerosols (CCN, IN) from Thompson microphysics -"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 -# Cloud effective radii from Thompson and WSM6 microphysics -"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 -# Prognostic/diagnostic variables from MYNN -"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 -# Prognostic/diagnostic variables from RUC LSM -"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -# Stochastic physics -"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"dynamics", "diss_est", "diss_est", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 - -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/field_table.FV3_CPT_v0 b/ush/templates/field_table.FV3_CPT_v0 deleted file mode 100644 index 7331ae4ae4..0000000000 --- a/ush/templates/field_table.FV3_CPT_v0 +++ /dev/null @@ -1,67 +0,0 @@ -# added by FRE: sphum must be present in atmos -# specific humidity for moist runs - "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=3.e-6" / -# prognostic cloud water mixing ratio - "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ice water mixing ratio - "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain water mixing ratio - "TRACER", "atmos_mod", "rainwat" - "longname", "rain water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic snow water mixing ratio - "TRACER", "atmos_mod", "snowwat" - "longname", "snow water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -#prognostic graupel water mixing ratio - "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water number concentration - "TRACER", "atmos_mod", "water_nc" - "longname", "cloud liquid water number concentration" - "units", "/m3" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud ice number concentration - "TRACER", "atmos_mod", "ice_nc" - "longname", "cloud ice water number concentration" - "units", "/m3" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic rain number concentration - "TRACER", "atmos_mod", "rain_nc" - "longname", "rain number concentration" - "units", "/m3" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic snow number concentration - "TRACER", "atmos_mod", "snow_nc" - "longname", "snow number concentration" - "units", "/m3" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic graupel number concentration - "TRACER", "atmos_mod", "graupel_nc" - "longname", "graupel number concentration" - "units", "/m3" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ozone mixing ratio tracer - "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic subgrid scale turbulent kinetic energy - "TRACER", "atmos_mod", "sgs_tke" - "longname", "subgrid scale turbulent kinetic energy" - "units", "m2/s2" - "profile_type", "fixed", "surface_value=0.0" / - diff --git a/ush/templates/field_table.FV3_GSD_SAR b/ush/templates/field_table.FV3_GSD_SAR deleted file mode 100644 index fe96567e5f..0000000000 --- a/ush/templates/field_table.FV3_GSD_SAR +++ /dev/null @@ -1,65 +0,0 @@ -# added by FRE: sphum must be present in atmos -# specific humidity for moist runs - "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water mixing ratio - "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ice water mixing ratio - "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain water mixing ratio - "TRACER", "atmos_mod", "rainwat" - "longname", "rain water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic snow water mixing ratio - "TRACER", "atmos_mod", "snowwat" - "longname", "snow water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic graupel mixing ratio - "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water number concentration - "TRACER", "atmos_mod", "water_nc" - "longname", "cloud liquid water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud ice number concentration - "TRACER", "atmos_mod", "ice_nc" - "longname", "cloud ice water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic rain number concentration - "TRACER", "atmos_mod", "rain_nc" - "longname", "rain number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic ozone mixing ratio tracer - "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# water- and ice-friendly aerosols (Thompson) - "TRACER", "atmos_mod", "liq_aero" - "longname", "water-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / - "TRACER", "atmos_mod", "ice_aero" - "longname", "ice-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic subgrid scale turbulent kinetic energy - "TRACER", "atmos_mod", "sgs_tke" - "longname", "subgrid scale turbulent kinetic energy" - "units", "m2/s2" - "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/templates/field_table.FV3_GSD_SAR_v1 b/ush/templates/field_table.FV3_GSD_SAR_v1 deleted file mode 100644 index 0a927de455..0000000000 --- a/ush/templates/field_table.FV3_GSD_SAR_v1 +++ /dev/null @@ -1,65 +0,0 @@ -# added by FRE: sphum must be present in atmos -# specific humidity for moist runs - "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water mixing ratio - "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ice water mixing ratio - "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain water mixing ratio - "TRACER", "atmos_mod", "rainwat" - "longname", "rain water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic snow water mixing ratio - "TRACER", "atmos_mod", "snowwat" - "longname", "snow water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic graupel mixing ratio - "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water number concentration - "TRACER", "atmos_mod", "water_nc" - "longname", "cloud liquid water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud ice number concentration - "TRACER", "atmos_mod", "ice_nc" - "longname", "cloud ice water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain number concentration - "TRACER", "atmos_mod", "rain_nc" - "longname", "rain number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ozone mixing ratio tracer - "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# water- and ice-friendly aerosols (Thompson) - "TRACER", "atmos_mod", "liq_aero" - "longname", "water-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / - "TRACER", "atmos_mod", "ice_aero" - "longname", "ice-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic subgrid scale turbulent kinetic energy - "TRACER", "atmos_mod", "sgs_tke" - "longname", "subgrid scale turbulent kinetic energy" - "units", "m2/s2" - "profile_type", "fixed", "surface_value=1.e30" / diff --git a/ush/templates/field_table.FV3_GSD_v0 b/ush/templates/field_table.FV3_GSD_v0 deleted file mode 100644 index fe96567e5f..0000000000 --- a/ush/templates/field_table.FV3_GSD_v0 +++ /dev/null @@ -1,65 +0,0 @@ -# added by FRE: sphum must be present in atmos -# specific humidity for moist runs - "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water mixing ratio - "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ice water mixing ratio - "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain water mixing ratio - "TRACER", "atmos_mod", "rainwat" - "longname", "rain water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic snow water mixing ratio - "TRACER", "atmos_mod", "snowwat" - "longname", "snow water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic graupel mixing ratio - "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water number concentration - "TRACER", "atmos_mod", "water_nc" - "longname", "cloud liquid water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud ice number concentration - "TRACER", "atmos_mod", "ice_nc" - "longname", "cloud ice water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic rain number concentration - "TRACER", "atmos_mod", "rain_nc" - "longname", "rain number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic ozone mixing ratio tracer - "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# water- and ice-friendly aerosols (Thompson) - "TRACER", "atmos_mod", "liq_aero" - "longname", "water-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / - "TRACER", "atmos_mod", "ice_aero" - "longname", "ice-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic subgrid scale turbulent kinetic energy - "TRACER", "atmos_mod", "sgs_tke" - "longname", "subgrid scale turbulent kinetic energy" - "units", "m2/s2" - "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 5a410adc4f..1ea3a86b85 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -29,11 +29,8 @@ valid_vals_PREDEF_GRID_NAME=( \ "RRFS_NA_3km" \ ) valid_vals_CCPP_PHYS_SUITE=( \ -"FV3_CPT_v0" \ "FV3_GFS_2017_gfdlmp" \ "FV3_GFS_2017_gfdlmp_regional" \ -"FV3_GSD_SAR" \ -"FV3_GSD_v0" \ "FV3_GFS_v15p2" \ "FV3_GFS_v15_thompson_mynn_lam3km" \ "FV3_GFS_v16" \ From 432e1788c7293b767c45c4a41c0d97fb88e21241 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Mon, 21 Mar 2022 14:38:58 -0600 Subject: [PATCH 134/203] Update locations of static data on Cheyenne, Hera, and Jet (#701) ## DESCRIPTION OF CHANGES: Several paths in the machine-specific files point to locations in user paths or old locations of static data. This PR updates paths of static data in regional_workflow/ush/machine/ to point to the official, centralized locations on Cheyenne, Hera, and Jet. ## TESTS CONDUCTED: Ran the following suite of end-to-end tests on Cheyenne and Jet prior to the latest ufs-weather-model hash update. All passed. This list of tests was chosen because all of these tests are known to succeed on all tested platforms, and this tests a variety of input and boundary condition types. - grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 - grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 - grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR - grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta - grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta On Hera, I ran tests with the latest SRW hash, which included the updated weather model. Because of this, many tests could not be generated due to using old, removed CCPP suites (see issue #668). To get around this issue, I tested with the fixes from #697 incorporated into my branch. With those extra commits, all "get_extrn_ics" and "get_extrn_lbcs" tasks completed successfully, which indicates that all data is in its correct place. ## ISSUE (optional): Will resolve a few issues in #673, many remain however. --- ush/machine/cheyenne.sh | 4 ++-- ush/machine/hera.sh | 4 ++-- ush/machine/jet.sh | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index 599e463528..6cf61061a0 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -68,6 +68,6 @@ NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/nda MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_CAM/FV3LAM_pregen" -TEST_COMINgfs="/glade/scratch/ketefian/NCO_dirs/COMGFS" +TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/FV3LAM_pregen" +TEST_COMINgfs="/glade/p/ral/jntp/UFS_SRW_app/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index bbe65dd3db..bb0f773134 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -71,8 +71,8 @@ NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/nda MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" +TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" TEST_COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/develop/model_data" TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index 800a0a6928..dac90d5490 100644 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -79,6 +79,6 @@ RUN_CMD_FCST="srun" RUN_CMD_POST="srun" # Test Data Locations -TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" -TEST_COMINgfs="/lfs1/HFIP/hwrf-data/hafs-input/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" +TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/FV3LAM_pregen" +TEST_COMINgfs="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/staged_extrn_mdl_files" From f07f66da4fd0172d686f394341e0886dadf32762 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 22 Mar 2022 09:25:27 -0400 Subject: [PATCH 135/203] Add -1 to restart_interval (#703) --- ush/templates/model_configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/templates/model_configure b/ush/templates/model_configure index 1b0f7a815a..fa2472e336 100644 --- a/ush/templates/model_configure +++ b/ush/templates/model_configure @@ -15,7 +15,7 @@ cpl: {{ cpl }} calendar: 'julian' memuse_verbose: .false. atmos_nthreads: {{ atmos_nthreads }} -restart_interval: {{ restart_interval }} +restart_interval: {{ restart_interval }} -1 output_1st_tstep_rst: .false. write_dopost: {{ write_dopost }} ideflate: 0 From d8dac8285f0ba7ad29faf068b12f6e43e08573ad Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 22 Mar 2022 09:26:21 -0400 Subject: [PATCH 136/203] back to original values for gfs_2017_gfdlmp (#706) --- ush/templates/FV3.input.yml | 1 + ush/templates/input.nml.FV3 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index d64a580792..f9256bc080 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -77,6 +77,7 @@ FV3_GFS_2017_gfdlmp: fv_debug: False k_split: 6 n_split: 6 + nord: 2 nord_zs_filter: !!python/none range_warn: False vtdm4: 0.075 diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index 6327120005..f94d00cbeb 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -119,7 +119,7 @@ rf_cutoff = 20.e2 tau = 5.0 use_hydro_pressure = .false. - vtdm4 = 0.075 + vtdm4 = 0.02 warm_start = .false. write_restart_with_bcs = .false. z_tracer = .true. From 598e38936796bf2adb6091778356c21c47fe0939 Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Tue, 22 Mar 2022 09:29:51 -0600 Subject: [PATCH 137/203] Convert verification to solely use METplus configuration files (feature/ens_design_RRFS) (#695) * Create feature/ens_design_RRFS branch and move original MET and METplua files to hold dirs. * Updated METplus conf files to remove dependency on project-based MET configs. * Updates to METplus conf files and run scripts for transitioning away from user-provided MET config files. * Removed MET configuration files; changed envar ACCUM back to acc in several GridStat conf files. * Removed old METplus conf files and removed OBS PROB info from Grid-Stat METplus conf files. --- scripts/exregional_run_ensgridvx.sh | 4 +- scripts/exregional_run_ensgridvx_mean.sh | 4 +- scripts/exregional_run_ensgridvx_prob.sh | 4 +- scripts/exregional_run_enspointvx.sh | 4 +- scripts/exregional_run_enspointvx_mean.sh | 4 +- scripts/exregional_run_enspointvx_prob.sh | 4 +- scripts/exregional_run_gridstatvx.sh | 8 +- scripts/exregional_run_pointstatvx.sh | 4 +- .../parm/met/EnsembleStatConfig_APCP | 284 --------------- .../parm/met/EnsembleStatConfig_REFC | 292 ---------------- .../parm/met/EnsembleStatConfig_point | 290 --------------- ush/templates/parm/met/GridDiagConfig | 42 --- ush/templates/parm/met/GridStatConfig_APCP | 180 ---------- .../parm/met/GridStatConfig_APCP_mean | 178 ---------- .../parm/met/GridStatConfig_APCP_prob | 178 ---------- ush/templates/parm/met/GridStatConfig_REFC | 219 ------------ .../parm/met/GridStatConfig_REFC_mean | 219 ------------ .../parm/met/GridStatConfig_REFC_prob | 219 ------------ ush/templates/parm/met/PB2NCConfig | 165 --------- ush/templates/parm/met/PointStatConfig | 258 -------------- ush/templates/parm/met/PointStatConfig_mean | 254 -------------- ush/templates/parm/met/PointStatConfig_prob | 254 -------------- ush/templates/parm/metplus/APCP_01h.conf | 99 ------ ush/templates/parm/metplus/APCP_03h.conf | 130 ------- ush/templates/parm/metplus/APCP_06h.conf | 129 ------- ush/templates/parm/metplus/APCP_24h.conf | 129 ------- .../parm/metplus/EnsembleStat_APCP01h.conf | 234 ++++++++++--- .../parm/metplus/EnsembleStat_APCP03h.conf | 262 ++++++++++---- .../parm/metplus/EnsembleStat_APCP06h.conf | 265 ++++++++++---- .../parm/metplus/EnsembleStat_APCP24h.conf | 267 ++++++++++---- .../parm/metplus/EnsembleStat_REFC.conf | 221 +++++++++--- .../parm/metplus/EnsembleStat_RETOP.conf | 231 +++++++++--- .../parm/metplus/EnsembleStat_conus_sfc.conf | 309 +++++++++++----- .../parm/metplus/EnsembleStat_upper_air.conf | 329 ++++++++++++------ ush/templates/parm/metplus/GridDiag_REFC.conf | 89 ----- .../parm/metplus/GridStat_APCP01h.conf | 274 +++++++++++++++ .../parm/metplus/GridStat_APCP01h_mean.conf | 289 ++++++++++++--- .../parm/metplus/GridStat_APCP01h_prob.conf | 274 ++++++++++++--- .../parm/metplus/GridStat_APCP03h.conf | 308 ++++++++++++++++ .../parm/metplus/GridStat_APCP03h_mean.conf | 289 ++++++++++++--- .../parm/metplus/GridStat_APCP03h_prob.conf | 275 ++++++++++++--- .../parm/metplus/GridStat_APCP06h.conf | 308 ++++++++++++++++ .../parm/metplus/GridStat_APCP06h_mean.conf | 289 ++++++++++++--- .../parm/metplus/GridStat_APCP06h_prob.conf | 277 ++++++++++++--- .../parm/metplus/GridStat_APCP24h.conf | 308 ++++++++++++++++ .../parm/metplus/GridStat_APCP24h_mean.conf | 289 ++++++++++++--- .../parm/metplus/GridStat_APCP24h_prob.conf | 279 ++++++++++++--- ush/templates/parm/metplus/GridStat_REFC.conf | 284 +++++++++++++++ .../parm/metplus/GridStat_REFC_mean.conf | 296 +++++++++++++--- .../parm/metplus/GridStat_REFC_prob.conf | 301 +++++++++++++--- .../parm/metplus/GridStat_RETOP.conf | 284 +++++++++++++++ .../parm/metplus/GridStat_RETOP_mean.conf | 299 +++++++++++++--- .../parm/metplus/GridStat_RETOP_prob.conf | 288 +++++++++++---- .../parm/metplus/PointStat_conus_sfc.conf | 225 +++++++++--- .../metplus/PointStat_conus_sfc_mean.conf | 222 +++++++++--- .../metplus/PointStat_conus_sfc_prob.conf | 222 +++++++++--- .../parm/metplus/PointStat_upper_air.conf | 227 +++++++++--- .../metplus/PointStat_upper_air_mean.conf | 224 +++++++++--- .../metplus/PointStat_upper_air_prob.conf | 224 +++++++++--- ush/templates/parm/metplus/REFC.conf | 102 ------ ush/templates/parm/metplus/RETOP.conf | 102 ------ ush/templates/parm/metplus/common.conf | 2 +- 62 files changed, 7182 insertions(+), 5341 deletions(-) delete mode 100755 ush/templates/parm/met/EnsembleStatConfig_APCP delete mode 100755 ush/templates/parm/met/EnsembleStatConfig_REFC delete mode 100644 ush/templates/parm/met/EnsembleStatConfig_point delete mode 100755 ush/templates/parm/met/GridDiagConfig delete mode 100755 ush/templates/parm/met/GridStatConfig_APCP delete mode 100755 ush/templates/parm/met/GridStatConfig_APCP_mean delete mode 100755 ush/templates/parm/met/GridStatConfig_APCP_prob delete mode 100644 ush/templates/parm/met/GridStatConfig_REFC delete mode 100644 ush/templates/parm/met/GridStatConfig_REFC_mean delete mode 100644 ush/templates/parm/met/GridStatConfig_REFC_prob delete mode 100644 ush/templates/parm/met/PB2NCConfig delete mode 100644 ush/templates/parm/met/PointStatConfig delete mode 100644 ush/templates/parm/met/PointStatConfig_mean delete mode 100644 ush/templates/parm/met/PointStatConfig_prob delete mode 100644 ush/templates/parm/metplus/APCP_01h.conf delete mode 100644 ush/templates/parm/metplus/APCP_03h.conf delete mode 100644 ush/templates/parm/metplus/APCP_06h.conf delete mode 100644 ush/templates/parm/metplus/APCP_24h.conf delete mode 100755 ush/templates/parm/metplus/GridDiag_REFC.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP01h.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP03h.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP06h.conf create mode 100644 ush/templates/parm/metplus/GridStat_APCP24h.conf create mode 100644 ush/templates/parm/metplus/GridStat_REFC.conf create mode 100644 ush/templates/parm/metplus/GridStat_RETOP.conf delete mode 100644 ush/templates/parm/metplus/REFC.conf delete mode 100644 ush/templates/parm/metplus/RETOP.conf diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh index 78cb5c2ba6..df1b91cd42 100755 --- a/scripts/exregional_run_ensgridvx.sh +++ b/scripts/exregional_run_ensgridvx.sh @@ -146,11 +146,11 @@ export LOG_SUFFIX # if [ ${VAR} == "APCP" ]; then acc="${ACCUM}h" # for stats output prefix in EnsembleStatConfig - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/EnsembleStat_${VAR}${acc}.conf else - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/EnsembleStat_${VAR}.conf fi diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh index b255adb868..bd50e3cff0 100755 --- a/scripts/exregional_run_ensgridvx_mean.sh +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -155,11 +155,11 @@ export LOG_SUFFIX # if [ ${VAR} == "APCP" ]; then export acc="${ACCUM}h" - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/GridStat_${VAR}${acc}_mean.conf else - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/GridStat_${VAR}_mean.conf fi diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh index 2ed9bdfead..7345c1728b 100755 --- a/scripts/exregional_run_ensgridvx_prob.sh +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -155,11 +155,11 @@ export LOG_SUFFIX # if [ ${VAR} == "APCP" ]; then export acc="${ACCUM}h" - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/GridStat_${VAR}${acc}_prob.conf else - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/GridStat_${VAR}_prob.conf fi diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh index 571711bf56..544f0f006d 100755 --- a/scripts/exregional_run_enspointvx.sh +++ b/scripts/exregional_run_enspointvx.sh @@ -137,11 +137,11 @@ export MODEL export NET export NUM_ENS_MEMBERS -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/EnsembleStat_conus_sfc.conf -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/EnsembleStat_upper_air.conf diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh index 4cd6039b9b..b65c9bd349 100755 --- a/scripts/exregional_run_enspointvx_mean.sh +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -138,11 +138,11 @@ export MET_CONFIG export MODEL export NET -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_conus_sfc_mean.conf -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_upper_air_mean.conf # diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh index 50d4e606fd..b315faac34 100755 --- a/scripts/exregional_run_enspointvx_prob.sh +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -138,11 +138,11 @@ export MET_CONFIG export MODEL export NET -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_conus_sfc_prob.conf -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_upper_air_prob.conf # diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh index e979cbe55b..63288c44ed 100755 --- a/scripts/exregional_run_gridstatvx.sh +++ b/scripts/exregional_run_gridstatvx.sh @@ -160,13 +160,13 @@ export NET # if [ ${VAR} == "APCP" ]; then export acc="${ACCUM}h" # for stats output prefix in GridStatConfig - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ - -c ${METPLUS_CONF}/${VAR}_${acc}.conf + -c ${METPLUS_CONF}/GridStat_${VAR}${acc}.conf else - ${METPLUS_PATH}/ush/master_metplus.py \ + ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ - -c ${METPLUS_CONF}/${VAR}.conf + -c ${METPLUS_CONF}/GridStat_${VAR}.conf fi # diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh index 705e086a8c..458b87749f 100755 --- a/scripts/exregional_run_pointstatvx.sh +++ b/scripts/exregional_run_pointstatvx.sh @@ -141,11 +141,11 @@ export MET_CONFIG export MODEL export NET -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_conus_sfc.conf -${METPLUS_PATH}/ush/master_metplus.py \ +${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ -c ${METPLUS_CONF}/PointStat_upper_air.conf diff --git a/ush/templates/parm/met/EnsembleStatConfig_APCP b/ush/templates/parm/met/EnsembleStatConfig_APCP deleted file mode 100755 index f9569105aa..0000000000 --- a/ush/templates/parm/met/EnsembleStatConfig_APCP +++ /dev/null @@ -1,284 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Ensemble-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BUDGET; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = []; -nc_var_str = ""; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product fields to be processed -// -ens = { - ens_thresh = ${ENS_THRESH}; - vld_thresh = 1.0; - - field = [ ${ENS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood ensemble probabilities -// -nbrhd_prob = { - width = [ 5 ]; - shape = CIRCLE; - vld_thresh = 0.0; -} - -// -// NMEP smoothing methods -// -nmep_smooth = { - vld_thresh = 0.0; - shape = CIRCLE; - gaussian_dx = 81.27; - gaussian_radius = 120; - type = [ - { - method = GAUSSIAN; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality = []; -duplicate_flag = UNIQUE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = TRUE; - -// -// Observation error options -// Set dist_type to NONE to use the observation error table instead -// May be set separately in each "obs.field" entry -// -obs_error = { - flag = TRUE; // TRUE or FALSE - dist_type = NONE; // Distribution type - dist_parm = []; // Distribution parameters - inst_bias_scale = 1.0; // Instrument bias scale adjustment - inst_bias_offset = 0.0; // Instrument bias offset adjustment - min = NA; // Valid range of data - max = NA; -} - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } -]; - -// -// Ensemble bin sizes -// May be set separately in each "obs.field" entry -// -ens_ssvar_bin_size = 1.0; -ens_phist_bin_size = 0.05; - -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = DW_MEAN; - day_interval = 31; - hour_interval = 6; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -// -// May be set separately in each "obs.field" entry -// -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; - write_bins = TRUE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - ${METPLUS_MASK_GRID} - ${METPLUS_MASK_POLY} - sid = []; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.01 ]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - type = [ { method = NEAREST; width = 1; } ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - ecnt = STAT; - rps = NONE; - rhist = STAT; - phist = STAT; - orank = STAT; - ssvar = STAT; - relp = STAT; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product output types -// -ensemble_flag = { - latlon = TRUE; - mean = TRUE; - stdev = TRUE; - minus = FALSE; - plus = FALSE; - min = FALSE; - max = FALSE; - range = FALSE; - vld_count = TRUE; - frequency = TRUE; - nep = FALSE; - nmep = FALSE; - rank = TRUE; - weight = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Random number generator -// -rng = { - type = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/EnsembleStatConfig_REFC b/ush/templates/parm/met/EnsembleStatConfig_REFC deleted file mode 100755 index 385e01a987..0000000000 --- a/ush/templates/parm/met/EnsembleStatConfig_REFC +++ /dev/null @@ -1,292 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Ensemble-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BUDGET; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = []; -nc_var_str = ""; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product fields to be processed -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -ens = { - ens_thresh = ${ENS_THRESH}; - vld_thresh = 1.0; - - field = [ ${ENS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood ensemble probabilities -// -nbrhd_prob = { - width = [ 5 ]; - shape = CIRCLE; - vld_thresh = 0.0; -} - -// -// NMEP smoothing methods -// -nmep_smooth = { - vld_thresh = 0.0; - shape = CIRCLE; - gaussian_dx = 81.27; - gaussian_radius = 120; - type = [ - { - method = GAUSSIAN; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Forecast and observation fields to be verified -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality = []; -duplicate_flag = UNIQUE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = TRUE; - -// -// Observation error options -// Set dist_type to NONE to use the observation error table instead -// May be set separately in each "obs.field" entry -// -obs_error = { - flag = FALSE; // TRUE or FALSE - dist_type = NONE; // Distribution type - dist_parm = []; // Distribution parameters - inst_bias_scale = 1.0; // Instrument bias scale adjustment - inst_bias_offset = 0.0; // Instrument bias offset adjustment - min = NA; // Valid range of data - max = NA; -} - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } -]; - -// -// Ensemble bin sizes -// May be set separately in each "obs.field" entry -// -ens_ssvar_bin_size = 1.0; -ens_phist_bin_size = 0.05; - -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = DW_MEAN; - day_interval = 31; - hour_interval = 6; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -// -// May be set separately in each "obs.field" entry -// -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; - write_bins = TRUE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - ${METPLUS_MASK_GRID} - ${METPLUS_MASK_POLY} - sid = []; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.01 ]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - type = [ { method = NEAREST; width = 1; } ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - ecnt = STAT; - rps = NONE; - rhist = STAT; - phist = STAT; - orank = STAT; - ssvar = STAT; - relp = STAT; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product output types -// -ensemble_flag = { - latlon = TRUE; - mean = TRUE; - stdev = TRUE; - minus = FALSE; - plus = FALSE; - min = FALSE; - max = FALSE; - range = FALSE; - vld_count = TRUE; - frequency = TRUE; - nep = FALSE; - nmep = FALSE; - rank = TRUE; - weight = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Random number generator -// -rng = { - type = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/EnsembleStatConfig_point b/ush/templates/parm/met/EnsembleStatConfig_point deleted file mode 100644 index 648893e201..0000000000 --- a/ush/templates/parm/met/EnsembleStatConfig_point +++ /dev/null @@ -1,290 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Ensemble-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = []; -nc_var_str = ""; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product fields to be processed -// -ens = { - ens_thresh = 0.05; - vld_thresh = 1.0; - - field = [ ${ENS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood ensemble probabilities -// -nbrhd_prob = { - width = [ 5 ]; - shape = CIRCLE; - vld_thresh = 0.0; -} - -// -// NMEP smoothing methods -// -nmep_smooth = { - vld_thresh = 0.0; - shape = CIRCLE; - gaussian_dx = 81.27; - gaussian_radius = 120; - type = [ - { - method = GAUSSIAN; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; -} -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -${METPLUS_MESSAGE_TYPE} -sid_inc = []; -sid_exc = []; -obs_thresh = [ NA ]; -obs_quality = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; -skip_const = FALSE; - -// -// Observation error options -// Set dist_type to NONE to use the observation error table instead -// May be set separately in each "obs.field" entry -// -obs_error = { - flag = FALSE; // TRUE or FALSE - dist_type = NONE; // Distribution type - dist_parm = []; // Distribution parameters - inst_bias_scale = 1.0; // Instrument bias scale adjustment - inst_bias_offset = 0.0; // Instrument bias offset adjustment - min = NA; // Valid range of data - max = NA; -} - -// -// Mapping of message type group name to comma-separated list of values -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } -]; - -// -// Ensemble bin sizes -// May be set separately in each "obs.field" entry -// -ens_ssvar_bin_size = 1.0; -ens_phist_bin_size = 0.05; - -// -// Categorical thresholds to define ensemble probabilities -// May be set separately in each "fcst.field" entry -// -prob_cat_thresh = []; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = DW_MEAN; - day_interval = 31; - hour_interval = 6; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -// -// May be set separately in each "obs.field" entry -// -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; - write_bins = TRUE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - ${METPLUS_MASK_GRID} - ${METPLUS_MASK_POLY} - sid = []; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - ecnt = STAT; - rps = NONE; - rhist = STAT; - phist = STAT; - orank = STAT; - ssvar = STAT; - relp = STAT; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Ensemble product output types -// -ensemble_flag = { - latlon = TRUE; - mean = TRUE; - stdev = TRUE; - minus = FALSE; - plus = FALSE; - min = FALSE; - max = FALSE; - range = TRUE; - vld_count = TRUE; - frequency = TRUE; - nep = FALSE; - nmep = FALSE; - rank = TRUE; - weight = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Random number generator -// -rng = { - type = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridDiagConfig b/ush/templates/parm/met/GridDiagConfig deleted file mode 100755 index baa6c8d5bb..0000000000 --- a/ush/templates/parm/met/GridDiagConfig +++ /dev/null @@ -1,42 +0,0 @@ -///////////////////////////////////////////////////////////////////////////////// -// -// Grid-Diag configuration file. -// -// For additional information, please see the MET User's Guide. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Data description -// -${METPLUS_DESC} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Output grid -// -${METPLUS_REGRID_DICT} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -${METPLUS_CENSOR_THRESH} -${METPLUS_CENSOR_VAL} - -// -// Data fields -// -${METPLUS_DATA_DICT} - -${METPLUS_MASK_DICT} - -${METPLUS_MET_CONFIG_OVERRIDES} - -//////////////////////////////////////////////////////////////////////////////// - -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_APCP b/ush/templates/parm/met/GridStatConfig_APCP deleted file mode 100755 index 4fa85ad7ee..0000000000 --- a/ush/templates/parm/met/GridStatConfig_APCP +++ /dev/null @@ -1,180 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = STAT; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - nbrctc = STAT; - nbrcts = STAT; - nbrcnt = STAT; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_APCP_mean b/ush/templates/parm/met/GridStatConfig_APCP_mean deleted file mode 100755 index 6aa4c8cc3b..0000000000 --- a/ush/templates/parm/met/GridStatConfig_APCP_mean +++ /dev/null @@ -1,178 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = STAT; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - nbrctc = STAT; - nbrcts = STAT; - nbrcnt = STAT; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_APCP_prob b/ush/templates/parm/met/GridStatConfig_APCP_prob deleted file mode 100755 index d6ef8cb0ec..0000000000 --- a/ush/templates/parm/met/GridStatConfig_APCP_prob +++ /dev/null @@ -1,178 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - pct = STAT; - pstd = STAT; - pjc = STAT; - prc = STAT; - nbrctc = NONE; - nbrcts = NONE; - nbrcnt = NONE; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFC b/ush/templates/parm/met/GridStatConfig_REFC deleted file mode 100644 index ade839df7a..0000000000 --- a/ush/templates/parm/met/GridStatConfig_REFC +++ /dev/null @@ -1,219 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = []; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = NONE; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Fourier decomposition -// May be set separately in each "obs.field" entry -// -fourier = { - wave_1d_beg = []; - wave_1d_end = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Gradient statistics -// May be set separately in each "obs.field" entry -// -gradient = { - dx = [ 1 ]; - dy = [ 1 ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = STAT; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - vcnt = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - nbrctc = STAT; - nbrcts = STAT; - nbrcnt = STAT; - grad = NONE; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - fourier = FALSE; - gradient = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFC_mean b/ush/templates/parm/met/GridStatConfig_REFC_mean deleted file mode 100644 index ade839df7a..0000000000 --- a/ush/templates/parm/met/GridStatConfig_REFC_mean +++ /dev/null @@ -1,219 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = []; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = NONE; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Fourier decomposition -// May be set separately in each "obs.field" entry -// -fourier = { - wave_1d_beg = []; - wave_1d_end = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Gradient statistics -// May be set separately in each "obs.field" entry -// -gradient = { - dx = [ 1 ]; - dy = [ 1 ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = STAT; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - vcnt = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - nbrctc = STAT; - nbrcts = STAT; - nbrcnt = STAT; - grad = NONE; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - fourier = FALSE; - gradient = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/GridStatConfig_REFC_prob b/ush/templates/parm/met/GridStatConfig_REFC_prob deleted file mode 100644 index 61570c4938..0000000000 --- a/ush/templates/parm/met/GridStatConfig_REFC_prob +++ /dev/null @@ -1,219 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - vld_thresh = 0.5; - method = BUDGET; - width = 2; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -cat_thresh = []; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; - -// -// Forecast and observation fields to be verified -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -fcst = { - field = [ ${FCST_FIELD} ]; -} - -obs = { - field = [ ${OBS_FIELD} ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology mean data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - vld_thresh = 0.5; - method = NEAREST; - width = 1; - } - - time_interp_method = DW_MEAN; - match_day = FALSE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = []; - poly = [ ${VERIF_MASK} ]; - sid = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = NONE; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - shape = ${NEIGHBORHOOD_SHAPE}; - width = [ ${NEIGHBORHOOD_WIDTH} ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Fourier decomposition -// May be set separately in each "obs.field" entry -// -fourier = { - wave_1d_beg = []; - wave_1d_end = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Gradient statistics -// May be set separately in each "obs.field" entry -// -gradient = { - dx = [ 1 ]; - dy = [ 1 ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - vcnt = NONE; - pct = STAT; - pstd = STAT; - pjc = STAT; - prc = STAT; - eclv = NONE; - nbrctc = NONE; - nbrcts = NONE; - nbrcnt = NONE; - grad = NONE; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - fourier = FALSE; - gradient = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = NONE; -rank_corr_flag = FALSE; -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PB2NCConfig b/ush/templates/parm/met/PB2NCConfig deleted file mode 100644 index ded8359c4f..0000000000 --- a/ush/templates/parm/met/PB2NCConfig +++ /dev/null @@ -1,165 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// PB2NC configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// PrepBufr message type -// -message_type = ${PB2NC_MESSAGE_TYPE}; - -// -// Mapping of message type group name to comma-separated list of values -// Derive PRMSL only for SURFACE message types -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; } -]; - -// -// Mapping of input PrepBufr message types to output message types -// -message_type_map = []; - -// -// PrepBufr station ID -// -station_id = ${PB2NC_STATION_ID}; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation time window -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation retention regions -// -mask = { - grid = "${PB2NC_GRID}"; - poly = "${PB2NC_POLY}"; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observing location elevation -// -elevation_range = { - beg = -1000; - end = 100000; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Observation types -// -pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; - -in_report_type = []; - -instrument_type = []; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Vertical levels to retain -// -level_range = { - beg = 1; - end = 511; -} - -level_category = [0, 1, 4, 5, 6]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// BUFR variable names to retain or derive. -// Use obs_bufr_map to rename variables in the output. -// If empty, process all available variables. -// -obs_bufr_var = ${OBS_BUFR_VAR_LIST}; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Mapping of input BUFR variable names to output variables names. -// The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -// -obs_bufr_map = [ - { key = "HOVI"; val = "VIS"; }, - { key = "PMO"; val = "PRMSL"; }, - { key = "POB"; val = "PRES"; }, - { key = "QOB"; val = "SPFH"; }, - { key = "TOB"; val = "TMP"; }, - { key = "TDO"; val = "DPT"; }, - { key = "ZOB"; val = "HGT"; }, - { key = "UOB"; val = "UGRD"; }, - { key = "VOB"; val = "VGRD"; }, - { key = "PWO"; val = "PWAT"; }, - { key = "D_DPT"; val = "DPT"; }, - { key = "D_WDIR"; val = "WDIR"; }, - { key = "D_WIND"; val = "WIND"; }, - { key = "MXGS"; val = "GUST"; }, - { key = "D_RH"; val = "RH"; }, - { key = "D_MIXR"; val = "MIXR"; }, - { key = "D_PBL"; val = "PBL"; }, - { key = "D_CAPE"; val = "CAPE"; }, - { key = "CEILING"; val = "CEILING"; }, - { key = "D_PRMSL"; val = "PRMSL"; }, - { key = "TOCC"; val = "TCDC"; } -]; - -// -// Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB -// abbreviations in the output. This default map is appended to obs_bufr_map. -// This should not typically be overridden. -// -obs_prefbufr_map = []; - -//////////////////////////////////////////////////////////////////////////////// - -quality_mark_thresh = 9; -event_stack_flag = TOP; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Time periods for the summarization -// obs_var (string array) is added and works like grib_code (int array) -// when use_var_id is enabled and variable names are saved. -// -time_summary = { - flag = ${TIME_SUMMARY_FLAG}; - raw_data = FALSE; - beg = ${TIME_SUMMARY_BEG}; - end = ${TIME_SUMMARY_END}; - step = 3600; - width = 3600; - grib_code = []; - obs_var = ${TIME_SUMMARY_VAR_NAMES}; - type = ${TIME_SUMMARY_TYPES}; - vld_freq = 0; - vld_thresh = 0.0; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig b/ush/templates/parm/met/PointStatConfig deleted file mode 100644 index 384ee83a6f..0000000000 --- a/ush/templates/parm/met/PointStatConfig +++ /dev/null @@ -1,258 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Point-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -rank_corr_flag = FALSE; - -// -// Forecast and observation fields to be verified -// - -M_to_KFT(x) = x * 3.28084 * 0.001; -KM_to_KFT(x) = x * 3280.84 * 0.001; - -fcst = { - field = [ ${FCST_FIELD} ]; - }; - -obs = { - field = [ ${OBS_FIELD} ]; - }; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = ${POINT_STAT_MESSAGE_TYPE}; -sid_exc = []; -obs_quality = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, - { key = "LANDSF"; val = "ADPSFC,MSONET"; }, - { key = "WATERSF"; val = "SFCSHP"; } -]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = NEAREST; - match_month = TRUE; - match_day = TRUE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; -write_cdf_bins = FALSE; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Land/Sea mask -// For LANDSF message types, only use forecast grid points where land = TRUE. -// For WATERSF message types, only use forecast grid points where land = FALSE. -// land_mask.flag may be set separately in each "obs.field" entry. -// -land_mask = { - flag = FALSE; - file_name = []; - field = { name = "LAND"; level = "L0"; } - regrid = { method = NEAREST; width = 1; } - thresh = eq1; -} - -// -// Topography -// For SURFACE message types, only use observations where the topo - station -// elevation difference meets the use_obs_thresh threshold. -// For the observations kept, when interpolating forecast data to the -// observation location, only use forecast grid points where the topo - station -// difference meets the interp_fcst_thresh threshold. -// topo_mask.flag may be set separately in each "obs.field" entry. -// -topo_mask = { - flag = FALSE; - file_name = []; - field = { name = "TOPO"; level = "L0"; } - regrid = { method = BILIN; width = 2; } - use_obs_thresh = ge-100&&le100; - interp_fcst_thresh = ge-50&&le50; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// May be set separately in each "obs.field" entry -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// May be set separately in each "obs.field" entry -// -mask = { - grid = ${POINT_STAT_GRID}; - poly = ${POINT_STAT_POLY}; - sid = ${POINT_STAT_STATION_ID}; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// May be set separately in each "obs.field" entry -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// May be set separately in each "obs.field" entry -// -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = BILIN; - width = 2; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// HiRA verification method -// May be set separately in each "obs.field" entry -// -hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; - vld_thresh = 1.0; - cov_thresh = [ ==0.25 ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// May be set separately in each "obs.field" entry -// -output_flag = { - fho = STAT; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = STAT; - sal1l2 = NONE; - vl1l2 = STAT; - val1l2 = NONE; - vcnt = STAT; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - mpr = NONE; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_mean b/ush/templates/parm/met/PointStatConfig_mean deleted file mode 100644 index 0b935c5786..0000000000 --- a/ush/templates/parm/met/PointStatConfig_mean +++ /dev/null @@ -1,254 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Point-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -rank_corr_flag = FALSE; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; - }; - -obs = { - field = [ ${OBS_FIELD} ]; - }; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = ${POINT_STAT_MESSAGE_TYPE}; -sid_exc = []; -obs_quality = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, - { key = "LANDSF"; val = "ADPSFC,MSONET"; }, - { key = "WATERSF"; val = "SFCSHP"; } -]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = NEAREST; - match_month = TRUE; - match_day = TRUE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; -write_cdf_bins = FALSE; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Land/Sea mask -// For LANDSF message types, only use forecast grid points where land = TRUE. -// For WATERSF message types, only use forecast grid points where land = FALSE. -// land_mask.flag may be set separately in each "obs.field" entry. -// -land_mask = { - flag = FALSE; - file_name = []; - field = { name = "LAND"; level = "L0"; } - regrid = { method = NEAREST; width = 1; } - thresh = eq1; -} - -// -// Topography -// For SURFACE message types, only use observations where the topo - station -// elevation difference meets the use_obs_thresh threshold. -// For the observations kept, when interpolating forecast data to the -// observation location, only use forecast grid points where the topo - station -// difference meets the interp_fcst_thresh threshold. -// topo_mask.flag may be set separately in each "obs.field" entry. -// -topo_mask = { - flag = FALSE; - file_name = []; - field = { name = "TOPO"; level = "L0"; } - regrid = { method = BILIN; width = 2; } - use_obs_thresh = ge-100&&le100; - interp_fcst_thresh = ge-50&&le50; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// May be set separately in each "obs.field" entry -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// May be set separately in each "obs.field" entry -// -mask = { - grid = ${POINT_STAT_GRID}; - poly = ${POINT_STAT_POLY}; - sid = ${POINT_STAT_STATION_ID}; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// May be set separately in each "obs.field" entry -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// May be set separately in each "obs.field" entry -// -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = BILIN; - width = 2; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// HiRA verification method -// May be set separately in each "obs.field" entry -// -hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; - vld_thresh = 1.0; - cov_thresh = [ ==0.25 ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// May be set separately in each "obs.field" entry -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = STAT; - sl1l2 = STAT; - sal1l2 = NONE; - vl1l2 = STAT; - val1l2 = NONE; - vcnt = STAT; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - mpr = NONE; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/met/PointStatConfig_prob b/ush/templates/parm/met/PointStatConfig_prob deleted file mode 100644 index f21ba1aaad..0000000000 --- a/ush/templates/parm/met/PointStatConfig_prob +++ /dev/null @@ -1,254 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Point-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// May be set separately in each "field" entry -// -regrid = { - to_grid = ${REGRID_TO_GRID}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = [ NA ]; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -rank_corr_flag = FALSE; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; - }; - -obs = { - field = [ ${OBS_FIELD} ]; - }; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation filtering options -// May be set separately in each "obs.field" entry -// -message_type = ${POINT_STAT_MESSAGE_TYPE}; -sid_exc = []; -obs_quality = []; -duplicate_flag = NONE; -obs_summary = NONE; -obs_perc_value = 50; - -// -// Mapping of message type group name to comma-separated list of values. -// -message_type_group_map = [ - { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET"; }, - { key = "ANYAIR"; val = "AIRCAR,AIRCFT"; }, - { key = "ANYSFC"; val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; }, - { key = "ONLYSF"; val = "ADPSFC,SFCSHP"; }, - { key = "LANDSF"; val = "ADPSFC,MSONET"; }, - { key = "WATERSF"; val = "SFCSHP"; } -]; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = { - - file_name = []; - field = []; - - regrid = { - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = NEAREST; - match_month = TRUE; - match_day = TRUE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; -write_cdf_bins = FALSE; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Land/Sea mask -// For LANDSF message types, only use forecast grid points where land = TRUE. -// For WATERSF message types, only use forecast grid points where land = FALSE. -// land_mask.flag may be set separately in each "obs.field" entry. -// -land_mask = { - flag = FALSE; - file_name = []; - field = { name = "LAND"; level = "L0"; } - regrid = { method = NEAREST; width = 1; } - thresh = eq1; -} - -// -// Topography -// For SURFACE message types, only use observations where the topo - station -// elevation difference meets the use_obs_thresh threshold. -// For the observations kept, when interpolating forecast data to the -// observation location, only use forecast grid points where the topo - station -// difference meets the interp_fcst_thresh threshold. -// topo_mask.flag may be set separately in each "obs.field" entry. -// -topo_mask = { - flag = FALSE; - file_name = []; - field = { name = "TOPO"; level = "L0"; } - regrid = { method = BILIN; width = 2; } - use_obs_thresh = ge-100&&le100; - interp_fcst_thresh = ge-50&&le50; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Point observation time window -// May be set separately in each "obs.field" entry -// -obs_window = { - beg = ${OBS_WINDOW_BEGIN}; - end = ${OBS_WINDOW_END}; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// May be set separately in each "obs.field" entry -// -mask = { - grid = ${POINT_STAT_GRID}; - poly = ${POINT_STAT_POLY}; - sid = ${POINT_STAT_STATION_ID}; - llpnt = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// May be set separately in each "obs.field" entry -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Interpolation methods -// May be set separately in each "obs.field" entry -// -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = BILIN; - width = 2; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// HiRA verification method -// May be set separately in each "obs.field" entry -// -hira = { - flag = FALSE; - width = [ 2, 3, 4, 5 ]; - vld_thresh = 1.0; - cov_thresh = [ ==0.25 ]; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// May be set separately in each "obs.field" entry -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - vcnt = NONE; - pct = STAT; - pstd = STAT; - pjc = STAT; - prc = STAT; - eclv = NONE; - mpr = NONE; -} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "/tmp"; -output_prefix = "${OUTPUT_PREFIX}"; -version = "V10.0.0"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/ush/templates/parm/metplus/APCP_01h.conf b/ush/templates/parm/metplus/APCP_01h.conf deleted file mode 100644 index 4f3599d327..0000000000 --- a/ush/templates/parm/metplus/APCP_01h.conf +++ /dev/null @@ -1,99 +0,0 @@ -# Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} - -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h - -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_01h.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = GridStat - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A01 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A01 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/APCP_03h.conf b/ush/templates/parm/metplus/APCP_03h.conf deleted file mode 100644 index ce3f1f5db9..0000000000 --- a/ush/templates/parm/metplus/APCP_03h.conf +++ /dev/null @@ -1,130 +0,0 @@ -# Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -# Grid to Grid Precipitation - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine -OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h - -[filename_templates] -# format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h -FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} - -# ANLYS -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_03h.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = PcpCombine, GridStat - -# Run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = True -OBS_PCP_COMBINE_RUN = True - -# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) -FCST_PCP_COMBINE_METHOD = ADD -OBS_PCP_COMBINE_METHOD = ADD - -FCST_PCP_COMBINE_CONSTANT_INIT = True - -# Accumulation interval available in forecast data -FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 - -# Accumulation interval available in obs data -OBS_PCP_COMBINE_INPUT_ACCUMS = 01 -OBS_PCP_COMBINE_OUTPUT_ACCUM = 03 - -# If 'bucket' output already exists, skip the PcpCombine step for the data -PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True - -# Forecast data description variables -FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB -FCST_IS_PROB = false - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A03 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A03 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/APCP_06h.conf b/ush/templates/parm/metplus/APCP_06h.conf deleted file mode 100644 index 6773ddba8a..0000000000 --- a/ush/templates/parm/metplus/APCP_06h.conf +++ /dev/null @@ -1,129 +0,0 @@ -# Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[OUTPUT_BASE]} - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine -OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h - -[filename_templates] -# format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h -FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} - -# ANLYS -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_06h.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = PcpCombine, GridStat - -# Run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = True -OBS_PCP_COMBINE_RUN = True - -# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) -FCST_PCP_COMBINE_METHOD = ADD -OBS_PCP_COMBINE_METHOD = ADD - -FCST_PCP_COMBINE_CONSTANT_INIT = True - -# Accumulation interval available in forecast data -FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 - -# Accumulation interval available in obs data -OBS_PCP_COMBINE_INPUT_ACCUMS = 01 -OBS_PCP_COMBINE_OUTPUT_ACCUM = 06 - -# If 'bucket' output already exists, skip the PcpCombine step for the data -PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True - -# Forecast data description variables -FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB -FCST_IS_PROB = false - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A06 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A06 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/APCP_24h.conf b/ush/templates/parm/metplus/APCP_24h.conf deleted file mode 100644 index 8b5a6bb985..0000000000 --- a/ush/templates/parm/metplus/APCP_24h.conf +++ /dev/null @@ -1,129 +0,0 @@ -# Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[OUTPUT_BASE]} - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine -OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h - -[filename_templates] -# format of filenames -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h -FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} - -# ANLYS -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_24h.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = CCPA -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = PcpCombine, GridStat - -# Run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = True -OBS_PCP_COMBINE_RUN = True - -# Mode of pcp_combine to use (SUM, ADD, SUBTRACT) -FCST_PCP_COMBINE_METHOD = ADD -OBS_PCP_COMBINE_METHOD = ADD - -FCST_PCP_COMBINE_CONSTANT_INIT = True - -# Accumulation interval available in forecast data -FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 - -# Accumulation interval available in obs data -OBS_PCP_COMBINE_INPUT_ACCUMS = 01 -OBS_PCP_COMBINE_OUTPUT_ACCUM = 24 - -# If 'bucket' output already exists, skip the PcpCombine step for the data -PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True - -# Forecast data description variables -FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB -FCST_IS_PROB = false - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -FCST_VAR1_NAME = APCP -FCST_VAR1_LEVELS = A24 - -BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700,ge25.400 - -OBS_VAR1_NAME = APCP -OBS_VAR1_LEVELS = A24 - -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf index 9172b3c17b..23f6adae9d 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf @@ -1,41 +1,8 @@ -# Ensemble Stat Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} - -# output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h - -[filename_templates] -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_01h.conf +# Ensemble-Stat METplus Configuration [config] -# Configuration-related settings such as the process list, begin and end times, etc. + +## Configuration-related settings such as the process list, begin and end times, etc. PROCESS_LIST = EnsembleStat # Looping by times: steps through each 'task' in the PROCESS_LIST for each @@ -49,13 +16,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -63,9 +30,18 @@ LEAD_SEQ = {ENV[fhr_list]} # Used in the MET config file for: model, output_prefix MODEL = {ENV[MODEL]} +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = CCPA +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -80,28 +56,109 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST - -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_01h_{OBTYPE} - -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_01h_{OBTYPE} -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. -ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = + +ENSEMBLE_STAT_DUPLICATE_FLAG = UNIQUE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = TRUE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, @@ -118,7 +175,6 @@ FCST_VAR1_LEVELS = A01 FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; - # Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION @@ -127,3 +183,69 @@ OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 + + +[filename_templates] + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_01h.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf index fbe1f518ae..d67380ad80 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf @@ -1,60 +1,15 @@ -# Ensemble Stat Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -# Output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h - -[filename_templates] -# Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h - -OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h - -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_03h.conf +# Ensemble-Stat METplus Configuration [config] -# Configuration-related settings such as the process list, begin and end times, etc. + +## Configuration-related settings such as the process list, begin and end times, etc. PROCESS_LIST = PcpCombine, EnsembleStat # Looping by times: steps through each 'task' in the PROCESS_LIST for each # defined time, and repeats until all times have been evaluated. LOOP_ORDER = times +# Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times @@ -64,13 +19,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -79,6 +34,8 @@ LEAD_SEQ = {ENV[fhr_list]} MODEL = {ENV[MODEL]} FCST_NATIVE_DATA_TYPE = GRIB +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = CCPA OBS_NATIVE_DATA_TYPE = GRIB @@ -108,6 +65,13 @@ PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB FCST_IS_PROB = false +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -122,29 +86,110 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST - -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_03h_{OBTYPE} +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_03h_{OBTYPE} -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = - -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = + +ENSEMBLE_STAT_DUPLICATE_FLAG = UNIQUE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = TRUE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE + # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, # (optional) ENS_VARn_OPTION @@ -158,10 +203,8 @@ ENS_VAR1_THRESH = gt0.0,ge0.508,ge2.54,ge6.350 FCST_VAR1_NAME = APCP FCST_VAR1_LEVELS = A03 -# FIGURE OUT WHAT BIN SIZE SHOULD BE FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; - # Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION @@ -170,3 +213,84 @@ OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Input and output fcst data directories for pcp-combine +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +# Input and output obs data directories for pcp-combine +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03 + + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h + +# Input and output template for obs pcp-combine files +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_03h.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf index 168cb9b984..ae2f61f449 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf @@ -1,60 +1,15 @@ -# Ensemble Stat Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -# Output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h - -[filename_templates] -# Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h - -OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h - -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_06h.conf +# Ensemble-Stat METplus Configuration [config] -# Configuration-related settings such as the process list, begin and end times, etc. + +## Configuration-related settings such as the process list, begin and end times, etc. PROCESS_LIST = PcpCombine, EnsembleStat # Looping by times: steps through each 'task' in the PROCESS_LIST for each # defined time, and repeats until all times have been evaluated. LOOP_ORDER = times +# Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times @@ -64,13 +19,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -79,6 +34,8 @@ LEAD_SEQ = {ENV[fhr_list]} MODEL = {ENV[MODEL]} FCST_NATIVE_DATA_TYPE = GRIB +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = CCPA OBS_NATIVE_DATA_TYPE = GRIB @@ -108,6 +65,13 @@ PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB FCST_IS_PROB = false +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -122,37 +86,118 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_06h_{OBTYPE} +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_06h_{OBTYPE} -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP - -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = - -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt -# Ensemble Variables and levels as specified in the ens field dictionary + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = + +ENSEMBLE_STAT_DUPLICATE_FLAG = UNIQUE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = TRUE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE + +# Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A06 ENS_VAR1_THRESH = gt0.0,ge2.54,ge6.350,ge12.700 -# Forecast Variables and levels as specified in the fcst field dictionary +# Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION FCST_VAR1_NAME = APCP @@ -162,7 +207,7 @@ FCST_VAR1_LEVELS = A06 FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; -# Observation Variables and levels as specified in the obs field dictionary +# Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION OBS_VAR1_NAME = {FCST_VAR1_NAME} @@ -170,3 +215,83 @@ OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Input and output fcst data directories for pcp-combine +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +# Input and output obs data directories for pcp-combine +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 + + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h + +# Input and output template for obs pcp-combine files +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_06h.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf index 1243e1f54a..0b1d175af0 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf @@ -1,60 +1,15 @@ -# Ensemble Stat Grid to Grid Precipitation Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} - -OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} - -# Output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h - -[filename_templates] -# Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h - -OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h - -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_24h.conf +# Ensemble-Stat METplus Configuration [config] -# Configuration-related settings such as the process list, begin and end times, etc. + +## Configuration-related settings such as the process list, begin and end times, etc. PROCESS_LIST = PcpCombine, EnsembleStat # Looping by times: steps through each 'task' in the PROCESS_LIST for each # defined time, and repeats until all times have been evaluated. LOOP_ORDER = times +# Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times @@ -64,13 +19,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -79,6 +34,8 @@ LEAD_SEQ = {ENV[fhr_list]} MODEL = {ENV[MODEL]} FCST_NATIVE_DATA_TYPE = GRIB +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = CCPA OBS_NATIVE_DATA_TYPE = GRIB @@ -95,7 +52,7 @@ FCST_PCP_COMBINE_CONSTANT_INIT = True # Accumulation interval available in forecast data FCST_PCP_COMBINE_INPUT_ACCUMS = 01 -FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 # Accumulation interval available in observation data OBS_PCP_COMBINE_INPUT_ACCUMS = 01 @@ -108,6 +65,13 @@ PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB FCST_IS_PROB = false +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -122,37 +86,118 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST - -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_24h_{OBTYPE} +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_APCP - -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_APCP_24h_{OBTYPE} -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt -# Ensemble Variables and levels as specified in the ens field dictionary + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = + +ENSEMBLE_STAT_DUPLICATE_FLAG = UNIQUE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = TRUE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE + +# Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, # (optional) ENS_VARn_OPTION ENS_VAR1_NAME = APCP ENS_VAR1_LEVELS = A24 ENS_VAR1_THRESH = gt0.0,ge6.350,ge12.700,ge25.400 -# Forecast Variables and levels as specified in the fcst field dictionary +# Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION FCST_VAR1_NAME = APCP @@ -162,7 +207,7 @@ FCST_VAR1_LEVELS = A24 FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; -# Observation Variables and levels as specified in the obs field dictionary +# Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION OBS_VAR1_NAME = {FCST_VAR1_NAME} @@ -170,3 +215,83 @@ OBS_VAR1_LEVELS = {FCST_VAR1_LEVELS} OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Input and output fcst data directories for pcp-combine +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} + +# Input and output obs data directories for pcp-combine +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 + + +[filename_templates] +# Need to have PCPCombine output data to individual member directories. +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h + +# Input and output template for obs pcp-combine files +OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_APCP_24h.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_REFC.conf b/ush/templates/parm/metplus/EnsembleStat_REFC.conf index 393f6037bd..989341a550 100644 --- a/ush/templates/parm/metplus/EnsembleStat_REFC.conf +++ b/ush/templates/parm/metplus/EnsembleStat_REFC.conf @@ -1,38 +1,4 @@ -# Ensemble Stat Grid to Grid REFC Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} - -# output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/REFC - -[filename_templates] -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_REFC.conf +# Ensemble-Stat METplus Configuration [config] @@ -50,13 +16,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -64,9 +30,18 @@ LEAD_SEQ = {ENV[fhr_list]} # Used in the MET config file for: model, output_prefix MODEL = {ENV[MODEL]} +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = MRMS +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -81,17 +56,109 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_REFC_{OBTYPE} +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_REFC_{OBTYPE} -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_REFC +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. -ENSEMBLE_STAT_MET_OBS_ERR_TABLE = +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = + + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = ADPSFC + +ENSEMBLE_STAT_DUPLICATE_FLAG = NONE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = FALSE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = FULL + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, @@ -108,12 +175,76 @@ FCST_VAR1_LEVELS = L0 FCST_VAR1_OPTIONS = ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; - # Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION - OBS_VAR1_NAME = MergedReflectivityQCComposite OBS_VAR1_LEVELS = Z500 + OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/REFC + + +[filename_templates] + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_REFC.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf index a90a9c7549..b0c235a2e8 100644 --- a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf +++ b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf @@ -1,38 +1,4 @@ -# Ensemble Stat Grid to Grid RETOP Example - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} - -# Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} - -# Grid observation input dir for ensemble_stat -OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} - -# output directory for ensemble_stat -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {OUTPUT_BASE}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/RETOP - -[filename_templates] -# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members -# or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# Template to look for grid observations. -OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 - -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat - -ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_RETOP.conf +# Ensemble-Stat METplus Configuration [config] @@ -50,13 +16,13 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d%H # Start time for METplus run -INIT_BEG = {ENV[CDATE]} +INIT_BEG={ENV[CDATE]} # End time for METplus run -INIT_END = {ENV[CDATE]} +INIT_END={ENV[CDATE]} # Increment between METplus runs in seconds. Must be >= 60 -INIT_INCREMENT = 3600 +INIT_INCREMENT=3600 # List of forecast leads to process LEAD_SEQ = {ENV[fhr_list]} @@ -64,9 +30,18 @@ LEAD_SEQ = {ENV[fhr_list]} # Used in the MET config file for: model, output_prefix MODEL = {ENV[MODEL]} +ENSEMBLE_STAT_DESC = NA ;; not in other file + # Name to identify observation data in output OBTYPE = MRMS +#ENSEMBLE_STAT_DESC = # not in other file + +# The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf +#LOG_ENSEMBLE_STAT_VERBOSITY = 2 + OBS_ENSEMBLE_STAT_WINDOW_BEGIN = 0 OBS_ENSEMBLE_STAT_WINDOW_END = 0 @@ -81,17 +56,109 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = FCST +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_RETOP_{OBTYPE} +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_RETOP_{OBTYPE} -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_REFC +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped # ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. -ENSEMBLE_STAT_MET_OBS_ERR_TABLE = +ENSEMBLE_STAT_MET_OBS_ERR_TABLE = + + +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = FCST +ENSEMBLE_STAT_REGRID_METHOD = BUDGET +ENSEMBLE_STAT_REGRID_WIDTH = 2 +ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = ADPSFC + +ENSEMBLE_STAT_DUPLICATE_FLAG = NONE +ENSEMBLE_STAT_SKIP_CONST = TRUE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = FALSE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = FULL + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, @@ -99,20 +166,88 @@ ENSEMBLE_STAT_MET_OBS_ERR_TABLE = ENS_VAR1_NAME = RETOP ENS_VAR1_LEVELS = L0 ENS_VAR1_THRESH = ge20, ge30, ge40, ge50 -ENS_VAR1_OPTIONS = convert(x) = M_to_KFT(x); +# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; +ENS_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; -# Forecast Variables and levels as specified in the fcst field dictionary +# Forecast Variables and levels as specified in the fcst field dictionary # of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION FCST_VAR1_NAME = RETOP FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; +# convert from meters to kilofeet: M_to_KFT(x) = x * 3.28084 * 0.001; +FCST_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; -# Observation Variables and levels as specified in the obs field dictionary +# Observation Variables and levels as specified in the obs field dictionary # of the MET configuration file. Specify as OBS_VARn_NAME, OBS_VARn_LEVELS, # (optional) OBS_VARn_OPTION OBS_VAR1_NAME = EchoTop18 OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; +# convert from kilometers to kilofeet: KM_to_KFT(x) = x * 3280.84 * 0.001; +OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; ens_ssvar_bin_size = 50.0; ens_phist_bin_size = 0.05; + +[dir] +# Input and output data directories +INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} +OUTPUT_BASE = {ENV[EXPTDIR]} + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {OUTPUT_BASE}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP + + +[filename_templates] + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.ensgrid_RETOP.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index 2523fde2a3..ecdc7f1ac9 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -1,89 +1,65 @@ -# Ensemble Stat Grid to Point Precipitation Example +# Ensemble-Stat METplus Configuration -[dir] -# Input and output directories for pb2nc -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -# Input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} - -# Point observation input dir for ensemble_stat (can also set grid obs) -OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -# Output directory for ensemble_stat -OUTPUT_BASE = {ENV[EXPTDIR]} -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc - -[filename_templates] +[config] -# Input and output templates for pb2nc -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +## Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PB2NC, EnsembleStat -# Input template for EnsembleStat can also be defined using a single -# member with wildcard characters to find multiple files -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc +# Start time for METplus run +INIT_BEG={ENV[CDATE]} -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.conus_surface.conf +# End time for METplus run +INIT_END={ENV[CDATE]} -# Location of MET poly files -#ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT=3600 -[config] -# Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PB2NC, EnsembleStat +# List of forecast leads to process +LEAD_SEQ = {ENV[fhr_list]} -# Time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +ENSEMBLE_STAT_DESC = NA -# List of forecast leads to process. -LEAD_SEQ = {ENV[fhr_list]} +# Name to identify observation data in output +OBTYPE = NDAS -# Order of loops to process data - Options are times, processes -LOOP_ORDER = times +#ENSEMBLE_STAT_DESC = # The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf LOG_ENSEMBLE_STAT_VERBOSITY = 3 -# MET Configuration files for pb2nc -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - -# if True, pb2nc will skip processing a file if the output already exists -# used to speed up runs and reduce redundancy -PB2NC_SKIP_IF_OUTPUT_EXISTS = True - # Values to pass to pb2nc config file using environment variables of the same name. PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # False for no time summary, True otherwise PB2NC_TIME_SUMMARY_FLAG = False PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format @@ -91,7 +67,6 @@ PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize -# For both PB2NC and ensemble_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -109,38 +84,119 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = NONE +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_point - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -MODEL = {ENV[MODEL]} -OBTYPE = NDAS - -# Used in the MET config file -ENSEMBLE_STAT_MESSAGE_TYPE = ADPSFC +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = +# MET Configuration files for pb2nc and ensemble_stat +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# if True, pb2nc will skip processing a file if the output already exists +# used to speed up runs and reduce redundancy +PB2NC_SKIP_IF_OUTPUT_EXISTS = True -# MET_OBS_ERR_TABLE is not required. +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt -# Variables and levels as specified in the field dictionary of the MET -# configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, -# (optional) FCST_VARn_OPTION +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = NONE +#ENSEMBLE_STAT_REGRID_METHOD = BILIN +#ENSEMBLE_STAT_REGRID_WIDTH = 2 +#ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +#ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = ADPSFC + +ENSEMBLE_STAT_DUPLICATE_FLAG = NONE +ENSEMBLE_STAT_SKIP_CONST = FALSE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = FALSE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +#ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +#ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION ENS_VAR1_NAME = TMP ENS_VAR1_LEVELS = Z02 ENS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 @@ -169,13 +225,19 @@ ENS_VAR6_LEVELS = L0 ENS_VAR6_THRESH = <152, <1520, >=914 ENS_VAR6_OPTIONS = GRIB_lvl_typ = 215; desc = "CEILING"; + +# Forecast and observation variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION and OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION + FCST_VAR1_NAME = TMP FCST_VAR1_LEVELS = Z2 FCST_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 OBS_VAR1_NAME = TMP OBS_VAR1_LEVELS = Z2 OBS_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 -OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR2_NAME = DPT FCST_VAR2_LEVELS = Z2 @@ -183,7 +245,7 @@ FCST_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 OBS_VAR2_NAME = DPT OBS_VAR2_LEVELS = Z2 OBS_VAR2_THRESH = >=263, >=268, >=273, >=288, >=293, >=298 -OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR3_NAME = WIND FCST_VAR3_LEVELS = Z10 @@ -192,7 +254,7 @@ FCST_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V co OBS_VAR3_NAME = WIND OBS_VAR3_LEVELS = Z10 OBS_VAR3_THRESH = >=5, >=10, >=15 -OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR4_NAME = TCDC FCST_VAR4_LEVELS = L0 @@ -217,4 +279,75 @@ OBS_VAR6_LEVELS = L0 OBS_VAR6_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR6_THRESH = <152, <305, >=914 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} +[dir] +# Input and output directories for pb2nc +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +OUTPUT_BASE = {ENV[EXPTDIR]} +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {ENV[EXPTDIR]}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc + + +[filename_templates] + +# Input and output templates for pb2nc +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Template for where ensemble-stat output is written +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.conus_surface.conf + diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf index fb2070e87e..d25241e936 100644 --- a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -1,94 +1,65 @@ -# Ensemble Stat Grid to Point Precipitation Example +# Ensemble-Stat METplus Configuration -[dir] -# Input and output directories for pb2nc -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -# Input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} - -# Point observation input dir for ensemble_stat (can also set grid obs) -OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -# Output directory for ensemble_stat -OUTPUT_BASE = {ENV[EXPTDIR]} -ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/upper_air - -[filename_templates] - -# Input and output templates for pb2nc -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +[config] -# Input ensemble template - comma separated list of ensemble members -#FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = -# mem01/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, -# mem02/postprd/{ENV[NET]}.t{init?fmt=%H}z.bgdawpf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, +## Configuration-related settings such as the process list, begin and end times, etc. +PROCESS_LIST = PB2NC, EnsembleStat -# Input template for EnsembleStat can also be defined using a single -# member with wildcard characters to find multiple files -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +# Looping by times: steps through each 'task' in the PROCESS_LIST for each +# defined time, and repeats until all times have been evaluated. +LOOP_ORDER = times -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} +# LOOP_BY: Set to INIT to loop over initialization times +LOOP_BY = INIT -ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat +# Format of INIT_BEG and INT_END +INIT_TIME_FMT = %Y%m%d%H -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa +# Start time for METplus run +INIT_BEG={ENV[CDATE]} -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.upper_air.conf +# End time for METplus run +INIT_END={ENV[CDATE]} -# Location of MET poly files -#ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Increment between METplus runs in seconds. Must be >= 60 +INIT_INCREMENT=3600 -[config] -# Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PB2NC, EnsembleStat +# List of forecast leads to process +LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) -# Time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Used in the MET config file for: model, output_prefix +MODEL = {ENV[MODEL]} -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +ENSEMBLE_STAT_DESC = NA -# List of forecast leads to process. -LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) +# Name to identify observation data in output +OBTYPE = NDAS -# Order of loops to process data - Options are times, processes -LOOP_ORDER = times +#ENSEMBLE_STAT_DESC = # The MET ensemble_stat logging level +# 0 quiet to 5 loud, Verbosity setting for MET ensemble_stat output, 2 is default. +# This takes precendence over the general LOG_MET_VERBOSITY set in metplus_logging.conf LOG_ENSEMBLE_STAT_VERBOSITY = 3 -# MET Configuration files for pb2nc -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - -# if True, pb2nc will skip processing a file if the output already exists -# used to speed up runs and reduce redundancy -PB2NC_SKIP_IF_OUTPUT_EXISTS = True - # Values to pass to pb2nc config file using environment variables of the same name. PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # False for no time summary, True otherwise PB2NC_TIME_SUMMARY_FLAG = False PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format @@ -96,7 +67,6 @@ PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize -# For both PB2NC and ensemble_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -114,38 +84,119 @@ ENSEMBLE_STAT_N_MEMBERS = {ENV[NUM_ENS_MEMBERS]} # threshold for ratio of valid files to expected files to allow app to run ENSEMBLE_STAT_ENS_THRESH = 0.05 -# Used in the MET config file for: regrid to_grid field -ENSEMBLE_STAT_REGRID_TO_GRID = NONE - -ENSEMBLE_STAT_CONFIG_FILE = {CONFIG_DIR}/EnsembleStatConfig_point - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -MODEL = {ENV[MODEL]} -OBTYPE = NDAS +# ens.vld_thresh value in the MET config file +ENSEMBLE_STAT_ENS_VLD_THRESH = 1.0 -# Used in the MET config file -ENSEMBLE_STAT_MESSAGE_TYPE = ADPUPA +ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} -# Verification Masking regions -# Indicate which grid and polygon masking region, if applicable -ENSEMBLE_STAT_MASK_GRID = +# MET Configuration files for pb2nc and ensemble_stat +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped +ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped -# List of full path to poly masking files. NOTE: Only short lists of poly -# files work (those that fit on one line), a long list will result in an -# environment variable that is too long, resulting in an error. For long -# lists of poly masking files (i.e. all the mask files in the NCEP_mask -# directory), define these in the MET point_stat configuration file. -ENSEMBLE_STAT_MASK_POLY = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# if True, pb2nc will skip processing a file if the output already exists +# used to speed up runs and reduce redundancy +PB2NC_SKIP_IF_OUTPUT_EXISTS = True -# MET_OBS_ERR_TABLE is not required. +# ENSEMBLE_STAT_MET_OBS_ERR_TABLE is not required. # If the variable is not defined, or the value is not set # than the MET default is used. ENSEMBLE_STAT_MET_OBS_ERR_TABLE = {MET_BASE}/table_files/obs_error_table.txt -# Variables and levels as specified in the field dictionary of the MET -# configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, -# (optional) FCST_VARn_OPTION +# Used in the MET config file for: regrid to_grid field +ENSEMBLE_STAT_REGRID_TO_GRID = NONE +#ENSEMBLE_STAT_REGRID_METHOD = BILIN +#ENSEMBLE_STAT_REGRID_WIDTH = 2 +#ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 +#ENSEMBLE_STAT_REGRID_SHAPE = SQUARE + +ENSEMBLE_STAT_CENSOR_THRESH = +ENSEMBLE_STAT_CENSOR_VAL = + +#ENSEMBLE_STAT_NBRHD_PROB_WIDTH = 5 +#ENSEMBLE_STAT_NBRHD_PROB_SHAPE = CIRCLE +#ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH = 0.0 + +#ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH = 0.0 +#ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE = CIRCLE +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 +#ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 +#ENSEMBLE_STAT_NMEP_SMOOTH_METHOD = GAUSSIAN +#ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH = 1 + +ENSEMBLE_STAT_MESSAGE_TYPE = ADPUPA + +ENSEMBLE_STAT_DUPLICATE_FLAG = NONE +ENSEMBLE_STAT_SKIP_CONST = FALSE + +ENSEMBLE_STAT_OBS_ERROR_FLAG = FALSE + +ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE = 1.0 +ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE = 0.05 + +#ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_MEAN_FIELD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_MEAN_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_MEAN_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_MEAN_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_MEAN_HOUR_INTERVAL = 6 + +#ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME = +#ENSEMBLE_STAT_CLIMO_STDEV_FIELD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#ENSEMBLE_STAT_CLIMO_STDEV_REGRID_SHAPE = +#ENSEMBLE_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#ENSEMBLE_STAT_CLIMO_STDEV_MATCH_MONTH = +#ENSEMBLE_STAT_CLIMO_STDEV_DAY_INTERVAL = 31 +#ENSEMBLE_STAT_CLIMO_STDEV_HOUR_INTERVAL = 6 + + +#ENSEMBLE_STAT_CLIMO_CDF_BINS = 1 +#ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS = False +#ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS = True + +ENSEMBLE_STAT_MASK_GRID = + +ENSEMBLE_STAT_CI_ALPHA = 0.05 + +ENSEMBLE_STAT_INTERP_FIELD = BOTH +ENSEMBLE_STAT_INTERP_VLD_THRESH = 1.0 +ENSEMBLE_STAT_INTERP_SHAPE = SQUARE +ENSEMBLE_STAT_INTERP_METHOD = NEAREST +ENSEMBLE_STAT_INTERP_WIDTH = 1 + +ENSEMBLE_STAT_OUTPUT_FLAG_ECNT = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RPS = NONE +ENSEMBLE_STAT_OUTPUT_FLAG_RHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_PHIST = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_ORANK = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR = STAT +ENSEMBLE_STAT_OUTPUT_FLAG_RELP = STAT + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE + +# Ensemble Variables and levels as specified in the ens field dictionary +# of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, +# (optional) ENS_VARn_OPTION ENS_VAR1_NAME = TMP ENS_VAR1_LEVELS = P850 ENS_VAR1_THRESH = >=288, >=293, >=298 @@ -166,7 +217,7 @@ ENS_VAR5_NAME = DPT ENS_VAR5_LEVELS = P700 ENS_VAR5_THRESH = >=263, >=268, >=273 -ENS_VAR6_NAME = WIND +ENS_VAR6_NAME = WIND ENS_VAR6_LEVELS = P850 ENS_VAR6_THRESH = >=5, >=10, >=15 @@ -195,13 +246,18 @@ ENS_VAR12_NAME = HPBL ENS_VAR12_LEVELS = Z0 ENS_VAR12_THRESH = <500, <1500, >1500 +# Forecast and observation variables and levels as specified in the fcst field dictionary +# of the MET configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, +# (optional) FCST_VARn_OPTION and OBS_VARn_NAME, OBS_VARn_LEVELS, +# (optional) OBS_VARn_OPTION + FCST_VAR1_NAME = TMP FCST_VAR1_LEVELS = P850 FCST_VAR1_THRESH = >=288, >=293, >=298 OBS_VAR1_NAME = TMP OBS_VAR1_LEVELS = P850 OBS_VAR1_THRESH = >=288, >=293, >=298 -OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR1_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR2_NAME = TMP FCST_VAR2_LEVELS = P700 @@ -209,7 +265,7 @@ FCST_VAR2_THRESH = >=273, >=278, >=283 OBS_VAR2_NAME = TMP OBS_VAR2_LEVELS = P700 OBS_VAR2_THRESH = >=273, >=278, >=283 -OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR2_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR3_NAME = TMP FCST_VAR3_LEVELS = P500 @@ -217,7 +273,7 @@ FCST_VAR3_THRESH = >=258, >=263, >=268 OBS_VAR3_NAME = TMP OBS_VAR3_LEVELS = P500 OBS_VAR3_THRESH = >=258, >=263, >=268 -OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR4_NAME = DPT FCST_VAR4_LEVELS = P850 @@ -225,7 +281,7 @@ FCST_VAR4_THRESH = >=273, >=278, >=283 OBS_VAR4_NAME = DPT OBS_VAR4_LEVELS = P850 OBS_VAR4_THRESH = >=273, >=278, >=283 -OBS_VAR4_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR4_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR5_NAME = DPT FCST_VAR5_LEVELS = P700 @@ -233,7 +289,7 @@ FCST_VAR5_THRESH = >=263, >=286, >=273 OBS_VAR5_NAME = DPT OBS_VAR5_LEVELS = P700 OBS_VAR5_THRESH = >=263, >=286, >=273 -OBS_VAR5_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR5_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR6_NAME = WIND FCST_VAR6_LEVELS = P850 @@ -241,7 +297,7 @@ FCST_VAR6_THRESH = >=5, >=10, >=15 OBS_VAR6_NAME = WIND OBS_VAR6_LEVELS = P850 OBS_VAR6_THRESH = >=5, >=10, >=15 -OBS_VAR6_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR6_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR7_NAME = WIND FCST_VAR7_LEVELS = P700 @@ -249,7 +305,7 @@ FCST_VAR7_THRESH = >=10, >=15, >=20 OBS_VAR7_NAME = WIND OBS_VAR7_LEVELS = P700 OBSVAR7_THRESH = >=10, >=15, >=20 -OBS_VAR7_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR7_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR8_NAME = WIND FCST_VAR8_LEVELS = P500 @@ -257,7 +313,7 @@ FCST_VAR8_THRESH = >=15, >=21, >=26 OBS_VAR8_NAME = WIND OBS_VAR8_LEVELS = P500 OBS_VAR8_THRESH = >=15, >=21, >=26 -OBS_VAR8_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR8_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR9_NAME = WIND FCST_VAR9_LEVELS = P250 @@ -265,7 +321,7 @@ FCST_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 OBS_VAR9_NAME = WIND OBS_VAR9_LEVELS = P250 OBS_VAR9_THRESH = >=26, >=31, >=36, >=46, >=62 -OBS_VAR9_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR9_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR10_NAME = HGT FCST_VAR10_LEVELS = P500 @@ -273,7 +329,7 @@ FCST_VAR10_THRESH = >=5400, >=5600, >=5880 OBS_VAR10_NAME = HGT OBS_VAR10_LEVELS = P500 OBS_VAR10_THRESH = >=5400, >=5600, >=5880 -OBS_VAR10_OPTIONS = obs_error = { flag = TRUE; } +OBS_VAR10_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; inst_bias_scale = 1.0; inst_bias_offset = 0.0; min = NA; max = NA; } FCST_VAR11_NAME = CAPE FCST_VAR11_LEVELS = L0 @@ -292,4 +348,75 @@ OBS_VAR12_LEVELS = L0 OBS_VAR12_OPTIONS = desc = "TKE"; OBS_VAR12_THRESH = <500, <1500, >1500 -ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} +[dir] +# Input and output directories for pb2nc +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +# Forecast model input directory for ensemble_stat +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} + +# Point observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# Grid observation input dir for ensemble_stat +OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to EnsembleStat +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = + +# output directory for ensemble_stat +OUTPUT_BASE = {ENV[EXPTDIR]} +ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +# directory containing log files +LOG_DIR = {ENV[EXPTDIR]}/log + +# directory for staging data +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air + + +[filename_templates] + +# Input and output templates for pb2nc +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members +# or a single line, - filename wildcard characters may be used, ? or *. + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for point observations. +# Example precip24_2010010112.nc +OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = {PB2NC_OUTPUT_TEMPLATE} + +# Template to look for gridded observations. +# Example ST4.2010010112.24h +OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = + +ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to EnsembleStat relative to ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Template for where ensemble-stat output is written +ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat/metplus_final.upper_air.conf + diff --git a/ush/templates/parm/metplus/GridDiag_REFC.conf b/ush/templates/parm/metplus/GridDiag_REFC.conf deleted file mode 100755 index c71d6a1079..0000000000 --- a/ush/templates/parm/metplus/GridDiag_REFC.conf +++ /dev/null @@ -1,89 +0,0 @@ -# -# CONFIGURATION -# -[config] - -INPUT_BASE={ENV[INPUT_BASE]} -OUTPUT_BASE={ENV[OUTPUT_BASE]} - -# 'Tasks' to be run -PROCESS_LIST = GridDiag - -GRID_DIAG_RUNTIME_FREQ = RUN_ONCE -#other options include: RUN_ONCE_PER_INIT_OR_VALID, RUN_ONCE_PER_LEAD, RUN_ONCE_FOR_EACH - -LOOP_BY = INIT - -# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. -# see www.strftime.org for more information -# %Y%m%d%H expands to YYYYMMDDHH -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run - must match INIT_TIME_FMT -INIT_BEG={ENV[INIT_BEG]} - -# End time for METplus run - must match INIT_TIME_FMT -INIT_END={ENV[INIT_END]} - -# Increment between METplus runs (in seconds if no units are specified) -# Must be >= 60 seconds -INIT_INCREMENT = {ENV[INIT_INC]} - -# List of forecast leads to process for each run time (init or valid) -# In hours if units are not specified -# If unset, defaults to 0 (don't loop through forecast leads) -LEAD_SEQ = begin_end_incr({ENV[FHR_FIRST]},{ENV[FHR_LAST]},{ENV[FHR_INC]}) - -# Looping by times: steps through each 'task' in the PROCESS_LIST for each -# defined time, and repeats until all times have been evaluated. -#LOOP_ORDER = times -LOOP_ORDER = processes -# must be processes when uisng GRID_DIAG_RUNTIME_FREQ - -LOG_GRID_DIAG_VERBOSITY = 2 - -GRID_DIAG_DESCRIPTION = RRFS -# Name to identify model (forecast) data in output -MODEL = {ENV[MODEL]} - -# Configuration file -GRID_DIAG_CONFIG_FILE = {CONFIG_DIR}/GridDiagConfig - -BOTH_VAR1_NAME = REFC -BOTH_VAR1_LEVELS = L0 -BOTH_VAR1_OPTIONS = cnt_thresh = [ >15 ]; n_bins = 18; range = [-20, 70]; - -BOTH_VAR2_NAME = MergedReflectivityQCComposite -BOTH_VAR2_LEVELS = Z500 -BOTH_VAR2_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; n_bins = 18; range = [-20, 70]; - -# The following variables set values in the MET -# configuration file used by this example -# Leaving these values commented will use the value -# found in the default MET configuration file -GRID_DIAG_REGRID_TO_GRID = FCST -#GRID_DIAG_REGRID_METHOD = BUDGET -#GRID_DIAG_REGRID_WIDTH = 2 -#GRID_DIAG_REGRID_VLD_THRESH = 0.5 -#GRID_DIAG_REGRID_SHAPE = SQUARE - -# DIRECTORIES -# -[dir] - -GRID_DIAG_INPUT_DIR = {INPUT_BASE} - -GRID_DIAG_OUTPUT_DIR = {OUTPUT_BASE} - -[filename_templates] - -GRID_DIAG_INPUT_TEMPLATE = expt_dirs/RRFS_baseline_summer/{init?fmt=%Y%m%d%H}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2, obs_data/mrms/proc/{valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 - -GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{ENV[INIT_BEG]}-{ENV[INIT_END]}_f{ENV[FHR_FIRST]}-{ENV[FHR_LAST]}.nc -#when using RUN_ONCE -#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{init?fmt=%Y%m%d%H}.nc when using RUN_ONCE_PER_INIT_OR_VALID -#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{lead?fmt=%3H}.nc -# when using RUN_ONCE_PER_LEAD -#GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out_{MODEL}_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.nc when using RUN_ONCE_FOR_EACH - -GRID_DIAG_VERIFICATION_MASK_TEMPLATE = /scratch2/BMC/fv3lam/RRFS_baseline/static/MET/masks/CONUS.nc diff --git a/ush/templates/parm/metplus/GridStat_APCP01h.conf b/ush/templates/parm/metplus/GridStat_APCP01h.conf new file mode 100644 index 0000000000..240462dacd --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP01h.conf @@ -0,0 +1,274 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_01h.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 + +#GRID_STAT_GRID_WEIGHT_FLAG = + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = APCP + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file +FCST_VAR1_LEVELS = A01 + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = A01 + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf index c43d63ead6..26740647e6 100644 --- a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf @@ -1,29 +1,58 @@ -# Ensemble Mean Grid to Grid Precipitation Example +# Ensemble Mean GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,65 +60,215 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP01.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_mean +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_mean FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_01h_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +# Name of forecast variable 1 FCST_VAR1_NAME = APCP_01_A01_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = A01 +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge2.54 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS OBS_VAR1_LEVELS = A01 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf index 52065ba7a5..d507983f2c 100644 --- a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Precipitation Example +# Ensemble Prob GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,54 +60,45 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP01.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_prob FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_01h_{OBTYPE}_prob +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" -# +# Name of forecast variables +# thresholds in mm, equal to >0, .01",.05",.10" FCST_VAR1_NAME = APCP_01_A01_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A01 FCST_VAR1_THRESH = ==0.1 @@ -111,11 +131,155 @@ OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A01 OBS_VAR4_THRESH = >=2.54 +#FCST_GRID_STAT_FILE_TYPE = -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP03h.conf b/ush/templates/parm/metplus/GridStat_APCP03h.conf new file mode 100644 index 0000000000..f6d6290215 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP03h.conf @@ -0,0 +1,308 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = PcpCombine, GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +# # If True, run PCPCombine on forecast data +# observation equivalent OBS_PCP_COMBINE_RUN also exists +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# mode of PCPCombine to use (SUM, ADD, SUBTRACT, DERIVE, or CUSTOM) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 03 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# maximum forecast lead to allow when searching for model data to use in PCPCombine +# Default is a very large time (4000 years) so setting this to a valid maximum value can +# speed up execution time of numerous runs +FCST_PCP_COMBINE_MAX_FORECAST = 2d + +# keep initialization time constant +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_03h.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 + +#GRID_STAT_GRID_WEIGHT_FLAG = + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = APCP + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file +FCST_VAR1_LEVELS = A03 + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = A03 + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to PCPCombine and GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# directory containing observation input to PCPCombine and GridStat +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from PCPCombine and GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf index 67db9927d2..d912c7f8d9 100644 --- a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf @@ -1,29 +1,58 @@ -# Ensemble Mean Grid to Grid Precipitation Example +# Ensemble Mean GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,59 +60,215 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP03.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_mean +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_mean +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_03h_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 FCST_VAR1_NAME = APCP_A3_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = A3 +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 BOTH_VAR1_THRESH = gt0.0,ge0.508,ge2.54,ge6.350 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS OBS_VAR1_LEVELS = A3 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf index 90f16b5223..30cd5ad102 100644 --- a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Precipitation Example +# Ensemble Prob GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,47 +60,44 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP03.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_03h_{OBTYPE}_prob +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +# Name of forecast variables FCST_VAR1_NAME = APCP_A3_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A03 FCST_VAR1_THRESH = ==0.1 @@ -104,10 +130,155 @@ OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A03 OBS_VAR4_THRESH = >=6.350 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +#FCST_GRID_STAT_FILE_TYPE = + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True -FCST_PROB_IN_GRIB_PDS = False +FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a03h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP06h.conf b/ush/templates/parm/metplus/GridStat_APCP06h.conf new file mode 100644 index 0000000000..7c8d69d1a3 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP06h.conf @@ -0,0 +1,308 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = PcpCombine, GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +# # If True, run PCPCombine on forecast data +# observation equivalent OBS_PCP_COMBINE_RUN also exists +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# mode of PCPCombine to use (SUM, ADD, SUBTRACT, DERIVE, or CUSTOM) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 06 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# maximum forecast lead to allow when searching for model data to use in PCPCombine +# Default is a very large time (4000 years) so setting this to a valid maximum value can +# speed up execution time of numerous runs +FCST_PCP_COMBINE_MAX_FORECAST = 2d + +# keep initialization time constant +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_06h.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 + +#GRID_STAT_GRID_WEIGHT_FLAG = + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = APCP + +# List of levels to evaluate for forecast variable 1 +# A06 = 6 hour accumulation in GRIB file +FCST_VAR1_LEVELS = A06 + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = A06 + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to PCPCombine and GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# directory containing observation input to PCPCombine and GridStat +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from PCPCombine and GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf index ef2ddb9535..264321eef3 100644 --- a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf @@ -1,29 +1,58 @@ -# Ensemble Mean Grid to Grid Precipitation Example +# Ensemble Mean GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,59 +60,215 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP06.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_mean +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_mean +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_06h_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 FCST_VAR1_NAME = APCP_A6_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = A06 +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 BOTH_VAR1_THRESH = gt0.0,ge2.54,ge6.350,ge12.700 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS OBS_VAR1_LEVELS = A06 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf index 5f75e50975..0bfdf4eaa9 100644 --- a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Precipitation Example +# Ensemble Prob GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,47 +60,44 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP06.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_prob +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_06h_{OBTYPE}_prob +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +# Name of forecast variables FCST_VAR1_NAME = APCP_A6_ENS_FREQ_gt0.0 FCST_VAR1_LEVELS = A06 FCST_VAR1_THRESH = ==0.1 @@ -104,10 +130,155 @@ OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A06 OBS_VAR4_THRESH = >=12.700 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +#FCST_GRID_STAT_FILE_TYPE = + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True -FCST_PROB_IN_GRIB_PDS = False +FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a06h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP24h.conf b/ush/templates/parm/metplus/GridStat_APCP24h.conf new file mode 100644 index 0000000000..bcccfe8e18 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_APCP24h.conf @@ -0,0 +1,308 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = PcpCombine, GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +# # If True, run PCPCombine on forecast data +# observation equivalent OBS_PCP_COMBINE_RUN also exists +FCST_PCP_COMBINE_RUN = True +OBS_PCP_COMBINE_RUN = True + +# mode of PCPCombine to use (SUM, ADD, SUBTRACT, DERIVE, or CUSTOM) +FCST_PCP_COMBINE_METHOD = ADD +OBS_PCP_COMBINE_METHOD = ADD + +# Accumulation interval available in forecast data +FCST_PCP_COMBINE_INPUT_ACCUMS = 01 +FCST_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# Accumulation interval available in obs data +OBS_PCP_COMBINE_INPUT_ACCUMS = 01 +OBS_PCP_COMBINE_OUTPUT_ACCUM = 24 + +# If 'bucket' output already exists, skip the PcpCombine step for the data +PCP_COMBINE_SKIP_IF_OUTPUT_EXISTS = True + +# maximum forecast lead to allow when searching for model data to use in PCPCombine +# Default is a very large time (4000 years) so setting this to a valid maximum value can +# speed up execution time of numerous runs +FCST_PCP_COMBINE_MAX_FORECAST = 2d + +# keep initialization time constant +FCST_PCP_COMBINE_CONSTANT_INIT = True + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_24h.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 + +#GRID_STAT_GRID_WEIGHT_FLAG = + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB +FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB + +# Name to identify observation data in output +OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = APCP + +# List of levels to evaluate for forecast variable 1 +# A24 = 24 hour accumulation in GRIB file +FCST_VAR1_LEVELS = A24 + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 +BOTH_VAR1_THRESH = gt0.0,ge0.254,ge0.508,ge1.27,ge2.54,ge3.810,ge6.350,ge8.890,ge12.700,ge25.400 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = A24 + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{ENV[acc]}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to PCPCombine and GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} +FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} + +# directory containing observation input to PCPCombine and GridStat +OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} +OBS_GRID_STAT_INPUT_DIR = {OBS_PCP_COMBINE_OUTPUT_DIR} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from PCPCombine and GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf index ad1308e20f..eaf9d5fb30 100644 --- a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf +++ b/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf @@ -1,29 +1,58 @@ -# Ensemble Mean Grid to Grid Precipitation Example +# Ensemble Mean GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,59 +60,215 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.APCP24.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_mean +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_mean +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_24h_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 FCST_VAR1_NAME = APCP_A24_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = A24 +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +#FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 BOTH_VAR1_THRESH = gt0.0,ge6.350,ge12.700,ge25.400 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = APCP + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS OBS_VAR1_LEVELS = A24 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf index a3659ea7b9..397abdc42e 100644 --- a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf +++ b/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Precipitation Example +# Ensemble Prob GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,49 +60,46 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.APCP24.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +#GRID_STAT_INTERP_FIELD = BOTH +#GRID_STAT_INTERP_VLD_THRESH = 1.0 +#GRID_STAT_INTERP_SHAPE = SQUARE +#GRID_STAT_INTERP_TYPE_METHOD = NEAREST +#GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +#GRID_STAT_GRID_WEIGHT_FLAG = -# Model to verify -MODEL = {ENV[MODEL]}_prob +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]}_prob +FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = CCPA +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_APCP_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_24h_{OBTYPE}_prob +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = [NA]; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare -# thresholds in mm, equal to .01",.02",.05",.10",.25",.50",.75",1.0" +# Name of forecast variables FCST_VAR1_NAME = APCP_A24_ENS_FREQ_gt0.0 -FCST_VAR1_LEVELS = A24 +FCST_VAR1_LEVELS = A24 FCST_VAR1_THRESH = ==0.1 OBS_VAR1_NAME = APCP @@ -104,10 +130,155 @@ OBS_VAR4_NAME = APCP OBS_VAR4_LEVELS = A24 OBS_VAR4_THRESH = >=25.400 -# Neighborhood shape and widths -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE +#FCST_GRID_STAT_FILE_TYPE = + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = 0 +OBS_GRID_STAT_FILE_WINDOW_END = 0 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True -FCST_PROB_IN_GRIB_PDS = False +FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_APCP_{ENV[acc]}_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_GRID_STAT_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a24h + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_REFC.conf b/ush/templates/parm/metplus/GridStat_REFC.conf new file mode 100644 index 0000000000..7eade0cb1d --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_REFC.conf @@ -0,0 +1,284 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.REFC.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 + +GRID_STAT_GRID_WEIGHT_FLAG = NONE + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = REFC + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file +FCST_VAR1_LEVELS = L0 + +FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = MergedReflectivityQCComposite + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = Z500 + +OBS_VAR1_OPTIONS = censor_thresh = [eq-999, <-20]; censor_val = [-9999, -20]; cnt_thresh = [ >15 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 1, 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_REFC_mean.conf b/ush/templates/parm/metplus/GridStat_REFC_mean.conf index 098e2c0102..6fad42b133 100644 --- a/ush/templates/parm/metplus/GridStat_REFC_mean.conf +++ b/ush/templates/parm/metplus/GridStat_REFC_mean.conf @@ -1,30 +1,58 @@ -# Ensemble Mean Composite Reflectivity Example +# GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/REFC_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -32,63 +60,225 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.REFC.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +GRID_STAT_GRID_WEIGHT_FLAG = NONE -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_mean +#FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = MRMS OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# Forecast/Observation variable Information +# Name of forecast variable 1 FCST_VAR1_NAME = REFC_L0_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = L0 +#FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = MergedReflectivityQCComposite -OBS_VAR1_LEVELS = Z500 + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = Z500 + OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 -# Neighborhood shape and width -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False -# Forecast data description variables +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_REFC_prob.conf b/ush/templates/parm/metplus/GridStat_REFC_prob.conf index d6ad7452a2..997fe1274a 100644 --- a/ush/templates/parm/metplus/GridStat_REFC_prob.conf +++ b/ush/templates/parm/metplus/GridStat_REFC_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Composite Reflectivity Example +# GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/REFC_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,61 +60,71 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.REFC.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +GRID_STAT_GRID_WEIGHT_FLAG = NONE -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_prob -FCST_NATIVE_DATA_TYPE = GRIB +#FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx -OBTYPE = MRMS +# Name to identify observation data in output +OBTYPE = MRMS OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_prob +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare +# Name of forecast variable 1 FCST_VAR1_NAME = REFC_L0_ENS_FREQ_ge20 -FCST_VAR1_LEVELS = L0 + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file +FCST_VAR1_LEVELS = L0 + +#FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 FCST_VAR1_THRESH = ==0.1 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = MergedReflectivityQCComposite -OBS_VAR1_LEVELS = Z500 + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = Z500 OBS_VAR1_THRESH = >=20.0 OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 + FCST_VAR2_NAME = REFC_L0_ENS_FREQ_ge30 FCST_VAR2_LEVELS = L0 FCST_VAR2_THRESH = ==0.1 @@ -113,10 +152,158 @@ OBS_VAR4_LEVELS = Z500 OBS_VAR4_THRESH = >=50.0 OBS_VAR4_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; -# Neighborhood shape and widths +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_REFC_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_MCTC = NONE +GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/REFC_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_REFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_RETOP.conf b/ush/templates/parm/metplus/GridStat_RETOP.conf new file mode 100644 index 0000000000..4ef239d362 --- /dev/null +++ b/ush/templates/parm/metplus/GridStat_RETOP.conf @@ -0,0 +1,284 @@ +# GridStat METplus Configuration + +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] + +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat + +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT + +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} + +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 + +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} + +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {ENV[EXPTDIR]}/log + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.RETOP.conf + +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped + +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE + +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 + +GRID_STAT_GRID_WEIGHT_FLAG = NONE + +# Name to identify model (forecast) data in output +MODEL = {ENV[MODEL]} +FCST_NATIVE_DATA_TYPE = GRIB + +# Name to identify observation data in output +OBTYPE = MRMS +OBS_NATIVE_DATA_TYPE = GRIB + +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools + +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; + +# Name of forecast variable 1 +FCST_VAR1_NAME = RETOP + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file +FCST_VAR1_LEVELS = L0 + +FCST_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; cnt_thresh = [ >0 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 +BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 + +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 +OBS_VAR1_NAME = EchoTop18 + +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = Z500 + +OBS_VAR1_OPTIONS = convert(x) = x * 3280.84 * 0.001; censor_thresh = [<=-9.84252,eq-3.28084]; censor_val = [-9999,-16.4042]; cnt_thresh = [ >0 ]; cnt_logic = UNION; + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 1, 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic +FCST_IS_PROB = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_RETOP_mean.conf b/ush/templates/parm/metplus/GridStat_RETOP_mean.conf index 6755ff2ee6..b0b17dbbd0 100644 --- a/ush/templates/parm/metplus/GridStat_RETOP_mean.conf +++ b/ush/templates/parm/metplus/GridStat_RETOP_mean.conf @@ -1,30 +1,58 @@ -# Ensemble Mean Composite RETOP Example +# GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_mean +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -32,64 +60,225 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.RETOP.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +GRID_STAT_GRID_WEIGHT_FLAG = NONE -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_mean +#FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx +# Name to identify observation data in output OBTYPE = MRMS OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_mean -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_mean +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA + +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# Forecast/Observation variable Information +# Name of forecast variable 1 FCST_VAR1_NAME = RETOP_L0_ENS_MEAN + +# List of levels to evaluate for forecast variable 1 +# A03 = 3 hour accumulation in GRIB file FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); +FCST_VAR1_OPTIONS = convert(x) = x * 3.28084 * 0.001; + +# List of thresholds to evaluate for each name/level combination for +# forecast variable 1 BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 +#FCST_GRID_STAT_FILE_TYPE = + +# Name of observation variable 1 OBS_VAR1_NAME = EchoTop18 -OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +# List of levels to evaluate for observation variable 1 +# (*,*) is NetCDF notation - must include quotes around these values! +# must be the same length as FCST_VAR1_LEVELS +OBS_VAR1_LEVELS = Z500 + +OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; + +# List of thresholds to evaluate for each name/level combination for +# observation variable 1 +#OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +#OBS_GRID_STAT_FILE_TYPE = + +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 OBS_GRID_STAT_FILE_WINDOW_END = 300 -# Neighborhood shape and width -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE + +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False -# Forecast data description variables +# Set to true if forecast data is probabilistic FCST_IS_PROB = False +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_mean + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_NBRCTC = STAT +GRID_STAT_OUTPUT_FLAG_NBRCTS = STAT +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/GridStat_RETOP_prob.conf b/ush/templates/parm/metplus/GridStat_RETOP_prob.conf index 41741a067b..27142979c0 100644 --- a/ush/templates/parm/metplus/GridStat_RETOP_prob.conf +++ b/ush/templates/parm/metplus/GridStat_RETOP_prob.conf @@ -1,29 +1,58 @@ -# Ensemble Prob Grid to Grid Composite RETOP Example +# GridStat METplus Configuration -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +# section heading for [config] variables - all items below this line and +# before the next section heading correspond to the [config] section +[config] -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} +# List of applications to run - only GridStat for this case +PROCESS_LIST = GridStat -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} +# time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_BY = INIT -LOG_DIR = {OUTPUT_BASE}/log +# Format of INIT_BEG and INT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H -STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_prob +# Start time for METplus run - must match INIT_TIME_FMT +INIT_BEG = {ENV[CDATE]} -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc +# End time for METplus run - must match INIT_TIME_FMT +INIT_END = {ENV[CDATE]} -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds +INIT_INCREMENT = 3600 -GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) +LEAD_SEQ = {ENV[fhr_list]} -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly +# Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run +LOOP_ORDER = times + +# Run pcp_combine on forecast/obs data? +FCST_PCP_COMBINE_RUN = False +OBS_PCP_COMBINE_RUN = False + +# Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only +#LOG_GRID_STAT_VERBOSITY = 2 + +LOG_DIR = {OUTPUT_BASE}/log # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} @@ -31,52 +60,44 @@ LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.RETOP.conf -[config] -# List of applications to run -PROCESS_LIST = GridStat +# Location of MET config file to pass to GridStat +GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT +# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary +# See MET User's Guide for more information +GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_REGRID_VLD_THRESH = 0.5 +GRID_STAT_REGRID_METHOD = BUDGET +GRID_STAT_REGRID_WIDTH = 2 +GRID_STAT_REGRID_SHAPE = SQUARE -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H +GRID_STAT_INTERP_FIELD = NONE +GRID_STAT_INTERP_VLD_THRESH = 1.0 +GRID_STAT_INTERP_SHAPE = SQUARE +GRID_STAT_INTERP_TYPE_METHOD = NEAREST +GRID_STAT_INTERP_TYPE_WIDTH = 1 -# list of forecast leads to process. -# LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times +GRID_STAT_GRID_WEIGHT_FLAG = NONE -# Model to verify +# Name to identify model (forecast) data in output MODEL = {ENV[MODEL]}_prob -FCST_NATIVE_DATA_TYPE = GRIB +#FCST_NATIVE_DATA_TYPE = GRIB -# Set obtype to vx -OBTYPE = MRMS +# Name to identify observation data in output +OBTYPE = MRMS OBS_NATIVE_DATA_TYPE = GRIB -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC_prob -GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_prob +# set the desc value in the GridStat MET config file +GRID_STAT_DESC = NA -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False +# List of variables to compare in GridStat - FCST_VAR1 variables correspond +# to OBS_VAR1 variables +# Note [FCST/OBS/BOTH]_GRID_STAT_VAR_NAME can be used instead if different evaluations +# are needed for different tools -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST +GRID_STAT_MET_CONFIG_OVERRIDES = cat_thresh = []; cnt_thresh = [NA]; cnt_logic = UNION; wind_thresh = [NA]; wind_logic = UNION; ci_alpha = [0.05]; rank_corr_flag = FALSE; -# list of variables to compare +# Name of forecast variable 1 FCST_VAR1_NAME = RETOP_L0_ENS_FREQ_ge20 FCST_VAR1_LEVELS = L0 FCST_VAR1_THRESH = ==0.1 @@ -84,7 +105,7 @@ FCST_VAR1_THRESH = ==0.1 OBS_VAR1_NAME = EchoTop18 OBS_VAR1_LEVELS = Z500 OBS_VAR1_THRESH = >=20.0 -OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +OBS_VAR1_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; FCST_VAR2_NAME = RETOP_L0_ENS_FREQ_ge30 FCST_VAR2_LEVELS = L0 @@ -93,7 +114,7 @@ FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = EchoTop18 OBS_VAR2_LEVELS = Z500 OBS_VAR2_THRESH = >=30.0 -OBS_VAR2_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +OBS_VAR2_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; FCST_VAR3_NAME = RETOP_L0_ENS_FREQ_ge40 FCST_VAR3_LEVELS = L0 @@ -102,7 +123,7 @@ FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = EchoTop18 OBS_VAR3_LEVELS = Z500 OBS_VAR3_THRESH = >=40.0 -OBS_VAR3_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +OBS_VAR3_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; FCST_VAR4_NAME = RETOP_L0_ENS_FREQ_ge50 FCST_VAR4_LEVELS = L0 @@ -111,12 +132,161 @@ FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = EchoTop18 OBS_VAR4_LEVELS = Z500 OBS_VAR4_THRESH = >=50.0 -OBS_VAR4_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = KM_to_KFT(x); +OBS_VAR4_OPTIONS = censor_thresh = lt-20.0; censor_val = -20.0; cnt_thresh = [ >15 ]; cnt_logic = UNION; convert(x) = x * 3280.84 * 0.001; + + +#OBS_GRID_STAT_FILE_TYPE = -# Neighborhood shape and widths +# Time relative to valid time (in seconds) to allow files to be considered +# valid. Set both BEGIN and END to 0 to require the exact time in the filename +# Not used in this example. +#FCST_GRID_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_GRID_STAT_FILE_WINDOW_END = 0 +OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 +OBS_GRID_STAT_FILE_WINDOW_END = 300 + +# MET GridStat neighborhood values +# See the MET User's Guide GridStat section for more information +GRID_STAT_NEIGHBORHOOD_FIELD = BOTH + +# width value passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 5, 7 + +# shape value passed to nbrhd dictionary in the MET config file GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 3,5,7 -# Forecast data description variables +# cov thresh list passed to nbrhd dictionary in the MET config file +GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 + +# Set to true to run GridStat separately for each field specified +# Set to false to create one run of GridStat per run time that +# includes all fields specified. +GRID_STAT_ONCE_PER_FIELD = False + +# Set to true if forecast data is probabilistic FCST_IS_PROB = True FCST_PROB_IN_GRIB_PDS = False + +# Only used if FCST_IS_PROB is true - sets probabilistic threshold +FCST_GRID_STAT_PROB_THRESH = ==0.1 + +# Set to true if observation data is probabilistic +# Only used if configuring forecast data as the 'OBS' input +OBS_IS_PROB = false + +# Only used if OBS_IS_PROB is true - sets probabilistic threshold +OBS_GRID_STAT_PROB_THRESH = ==0.1 + +GRID_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_RETOP_{OBTYPE}_prob + +# Climatology data +#GRID_STAT_CLIMO_MEAN_FILE_NAME = +#GRID_STAT_CLIMO_MEAN_FIELD = +#GRID_STAT_CLIMO_MEAN_REGRID_METHOD = +#GRID_STAT_CLIMO_MEAN_REGRID_WIDTH = +#GRID_STAT_CLIMO_MEAN_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_MEAN_REGRID_SHAPE = +#GRID_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_MEAN_MATCH_MONTH = +#GRID_STAT_CLIMO_MEAN_DAY_INTERVAL = +#GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL = + +#GRID_STAT_CLIMO_STDEV_FILE_NAME = +#GRID_STAT_CLIMO_STDEV_FIELD = +#GRID_STAT_CLIMO_STDEV_REGRID_METHOD = +#GRID_STAT_CLIMO_STDEV_REGRID_WIDTH = +#GRID_STAT_CLIMO_STDEV_REGRID_VLD_THRESH = +#GRID_STAT_CLIMO_STDEV_REGRID_SHAPE = +#GRID_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = +#GRID_STAT_CLIMO_STDEV_MATCH_MONTH = +#GRID_STAT_CLIMO_STDEV_DAY_INTERVAL = +#GRID_STAT_CLIMO_STDEV_HOUR_INTERVAL = + +GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + +GRID_STAT_MASK_GRID = + +# Statistical output types +GRID_STAT_OUTPUT_FLAG_FHO = NONE +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_MCTC = NONE +GRID_STAT_OUTPUT_FLAG_MCTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = NONE +GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +GRID_STAT_OUTPUT_FLAG_VCNT = NONE +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_PJC = STAT +GRID_STAT_OUTPUT_FLAG_PRC = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +# NetCDF matched pairs output file +#GRID_STAT_NC_PAIRS_VAR_NAME = +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +# End of [config] section and start of [dir] section +[dir] + +# directory containing forecast input to GridStat +INPUT_BASE = {ENV[INPUT_BASE]} +FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} + +# directory containing observation input to GridStat +OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to GridStat +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_DIR = + +# directory to write output from GridStat +OUTPUT_BASE = {ENV[EXPTDIR]} +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR +FCST_GRID_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_RETOP_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR +OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 + +# Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from GridStat +GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +# Used to specify one or more verification mask files for GridStat +# Not used for this example +GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 779fa73b45..1558c4aa99 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -1,66 +1,52 @@ -# Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.conus_surface.conf - [config] -# List of applications to run +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -68,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -80,11 +63,19 @@ PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_WIND, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # For defining the time periods for summarization # False for no time summary, True otherwise # The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False @@ -96,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +POINT_STAT_OUTPUT_FLAG_FHO = STAT +POINT_STAT_OUTPUT_FLAG_CTC = STAT +POINT_STAT_OUTPUT_FLAG_CTS = STAT +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = STAT +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +POINT_STAT_OUTPUT_FLAG_VCNT = STAT +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]} + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {MODEL}_{OBTYPE}_ADPSFC +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -124,10 +175,18 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPSFC - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead + BOTH_VAR1_NAME = TMP BOTH_VAR1_LEVELS = Z2 @@ -147,7 +206,7 @@ BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts BOTH_VAR6_NAME = WIND BOTH_VAR6_LEVELS = Z10 -BOTH_VAR6_THRESH = >=2.572, >=2.572 && <5.144, >=5.144, >=10.288, >=15.433 ;; m/s or 5, 10, 20, 30kts +BOTH_VAR6_THRESH = >=2.572, >=2.572&&<5.144, >=5.144, >=10.288, >=15.433 ;; m/s or 5, 10, 20, 30kts BOTH_VAR6_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V components, overriding max 10-m wind BOTH_VAR7_NAME = PRMSL @@ -155,7 +214,7 @@ BOTH_VAR7_LEVELS = Z0 FCST_VAR8_NAME = TCDC FCST_VAR8_LEVELS = L0 -BOTH_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } +FCST_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR8_NAME = TCDC OBS_VAR8_LEVELS = L0 OBS_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } @@ -176,3 +235,63 @@ OBS_VAR11_NAME = CEILING OBS_VAR11_LEVELS = L0 OBS_VAR11_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR11_THRESH = <152, <305, <914, <1520, <3040, >=914 + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.conus_surface.conf diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf index fcc45a4c9a..db3b3f9329 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf @@ -1,64 +1,52 @@ -# Ensemble Mean Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[EXPTDIR]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_mean - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.conus_surface.conf - [config] -# List of applications to run + +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -66,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -77,12 +62,20 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # For defining the time periods for summarization # False for no time summary, True otherwise # The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False @@ -94,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_mean +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = STAT +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +POINT_STAT_OUTPUT_FLAG_VCNT = STAT +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]}_mean + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPSFC_{OBTYPE}_mean +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -122,10 +175,17 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPSFC - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_Z2_ENS_MEAN FCST_VAR1_LEVELS = Z2 FCST_VAR1_THRESH = >=268, >=273, >=278, >=293, >=298, >=303 @@ -149,3 +209,63 @@ FCST_VAR3_THRESH = >=5, >=10, >=15 OBS_VAR3_NAME = WIND OBS_VAR3_LEVELS = Z10 OBS_VAR3_THRESH = >=5, >=10, >=15 + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.conus_surface.conf diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf index bffc327add..a1754261e7 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -1,64 +1,52 @@ -# Ensemble Prob Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[EXPTDIR]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_prob - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.conus_surface.conf - [config] -# List of applications to run + +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -66,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -77,12 +62,20 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # For defining the time periods for summarization # False for no time summary, True otherwise # The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False @@ -94,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_prob +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +#POINT_STAT_OUTPUT_FLAG_CNT = +#POINT_STAT_OUTPUT_FLAG_SL1L2 = +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VL1L2 = +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VCNT = +POINT_STAT_OUTPUT_FLAG_PCT = STAT +POINT_STAT_OUTPUT_FLAG_PSTD = STAT +POINT_STAT_OUTPUT_FLAG_PJC = STAT +POINT_STAT_OUTPUT_FLAG_PRC = STAT +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]}_prob + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPSFC_{OBTYPE}_prob +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -122,10 +175,17 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPSFC - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_Z2_ENS_FREQ_ge268 FCST_VAR1_LEVELS = (*,*) FCST_VAR1_THRESH = ==0.1 @@ -325,3 +385,63 @@ OBS_VAR23_OPTIONS = GRIB_lvl_typ = 215; interp = { type = [ { method = NEAREST; # Forecast data description variables FCST_IS_PROB = True FCST_PROB_IN_GRIB_PDS = False + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC_INPUT_DIR = {ENV[OBS_DIR]} +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.conus_surface.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index ca60b7d5d0..e21f2566c0 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -1,66 +1,52 @@ -# Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/upper_air - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} - -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.upper_air.conf - [config] -# List of applications to run + +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) -#LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -68,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -80,10 +63,18 @@ PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, D_WIND, HOVI, CEILING, D_PBL, D_CAPE, MXGS +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_WIND, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS + +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; # For defining the time periods for summarization # False for no time summary, True otherwise @@ -96,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +POINT_STAT_OUTPUT_FLAG_FHO = STAT +POINT_STAT_OUTPUT_FLAG_CTC = STAT +POINT_STAT_OUTPUT_FLAG_CTS = STAT +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = STAT +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +POINT_STAT_OUTPUT_FLAG_VCNT = STAT +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]} + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {MODEL}_{OBTYPE}_ADPUPA +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -124,10 +175,17 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPUPA - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead BOTH_VAR1_NAME = TMP BOTH_VAR1_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 @@ -147,7 +205,7 @@ BOTH_VAR5_THRESH = >=2.572 ;; m/s or 5kts BOTH_VAR6_NAME = WIND BOTH_VAR6_LEVELS = P1000, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 -BOTH_VAR6_THRESH = >=2.572, >=2.572 && <5.144, >=5.144, >=10.288, >=15.433, >=20.577, >=25.722 ;; m/s or 5, 10, 20, 30, 40, 50kts +BOTH_VAR6_THRESH = >=2.572, >=2.572&&<5.144, >=5.144, >=10.288, >=15.433, >=20.577, >=25.722 ;; m/s or 5, 10, 20, 30, 40, 50kts BOTH_VAR7_NAME = HGT BOTH_VAR7_LEVELS = P1000, P950, P925, P850, P700, P500, P400, P300, P250, P200, P150, P100, P50, P20, P10 @@ -176,3 +234,64 @@ FCST_VAR11_OPTIONS = GRIB_lvl_typ = 220; OBS_VAR11_NAME = PBL OBS_VAR11_LEVELS = L0 OBS_VAR11_OPTIONS = desc = "RI"; + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[OUTPUT_BASE]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} + +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.upper_air.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf index 3f7dfc8f27..5152536831 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -1,65 +1,52 @@ -# Ensemble Mean Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[EXPTDIR]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_mean - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.upper_air.conf - [config] -# List of applications to run + +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) -#LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -67,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -78,12 +62,20 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # For defining the time periods for summarization # False for no time summary, True otherwise # The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False @@ -95,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_mean +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +POINT_STAT_OUTPUT_FLAG_CNT = STAT +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +POINT_STAT_OUTPUT_FLAG_VCNT = STAT +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]}_mean + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPUPA_{OBTYPE}_mean +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -123,10 +175,17 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPUPA - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_P850_ENS_MEAN FCST_VAR1_LEVELS = P850 FCST_VAR1_THRESH = >=288, >=293, >=298 @@ -225,3 +284,64 @@ OBS_VAR12_NAME = PBL OBS_VAR12_LEVELS = L0 OBS_VAR12_OPTIONS = desc = "TKE"; OBS_VAR12_THRESH = <500, <1500, >1500 + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_mean + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} + +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_mean + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_mean/metplus_final.upper_air.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf index cb653746b1..0a15f6f1f0 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -1,65 +1,52 @@ -# Ensemble Prob Grid to Point Example - -[dir] -# Input and output data directories -PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc - -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} - -OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} - -OUTPUT_BASE = {ENV[EXPTDIR]} -POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_prob - -[filename_templates] -# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} -# Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -# Template to look for forecast/observation input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc - -POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.upper_air.conf - [config] -# List of applications to run + +# List of applications to run - only PointStat for this case PROCESS_LIST = PB2NC, PointStat # time looping - options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set LOOP_BY = INIT -# Start time for METplus run +# Format of INIT_BEG and INIT_END using % items +# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. +# see www.strftime.org for more information +# %Y%m%d%H expands to YYYYMMDDHH +INIT_TIME_FMT = %Y%m%d%H + +# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG = {ENV[CDATE]} -# End time for METplus run + +# End time for METplus run - must match INIT_TIME_FMT INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 + +# Increment between METplus runs (in seconds if no units are specified) +# Must be >= 60 seconds INIT_INCREMENT = 3600 -INIT_TIME_FMT = %Y%m%d%H -# list of forecast leads to process. +# List of forecast leads to process for each run time (init or valid) +# In hours if units are not specified +# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = begin_end_incr(0,{ENV[fhr_last]},6) -#LEAD_SEQ = {ENV[fhr_list]} # Order of loops to process data - Options are times, processes +# Not relevant if only one item is in the PROCESS_LIST +# times = run all wrappers in the PROCESS_LIST for a single run time, then +# increment the run time and run all wrappers again until all times have +# been evaluated. +# processes = run the first wrapper in the PROCESS_LIST for all times +# specified, then repeat for the next item in the PROCESS_LIST until all +# wrappers have been run LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for PointStat only LOG_POINT_STAT_VERBOSITY = 2 +# Location of MET config file to pass to PB2NC +PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped + # For both PB2NC and point_stat OBS_WINDOW_BEGIN = -1799 OBS_WINDOW_END = 1800 @@ -67,9 +54,6 @@ OBS_WINDOW_END = 1800 PB2NC_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} PB2NC_WINDOW_END = {OBS_WINDOW_END} -# Location of MET config file to pass to PB2NC -PB2NC_CONFIG_FILE = {CONFIG_DIR}/PB2NCConfig - # If set to True, skip run if the output file determined by the output directory and # filename template already exists PB2NC_SKIP_IF_OUTPUT_EXISTS = True @@ -78,12 +62,20 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_GRID = PB2NC_POLY = PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = +PB2NC_MESSAGE_TYPE = ADPSFC, ADPUPA +PB2NC_LEVEL_CATEGORY = 0, 1, 4, 5, 6 +PB2NC_QUALITY_MARK_THRESH = 9 + +PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all # PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +# Mapping of input BUFR variable names to output variables names. +# The default PREPBUFR map, obs_prepbufr_map, is appended to this map. +PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; + # For defining the time periods for summarization # False for no time summary, True otherwise # The rest of the PB2NC_TIME_SUMMARY variables are ignored if set to False @@ -95,22 +87,82 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # Location of MET config file to pass to GridStat # References PARM_BASE which is the location of the parm directory corresponding -# to the ush directory of the master_metplus.py script that is called +# to the ush directory of the run_metplus.py script that is called # or the value of the environment variable METPLUS_PARM_BASE if set -POINT_STAT_CONFIG_FILE ={CONFIG_DIR}/PointStatConfig_prob +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = + +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +#POINT_STAT_OUTPUT_FLAG_CNT = +#POINT_STAT_OUTPUT_FLAG_SL1L2 = +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VL1L2 = +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VCNT = +POINT_STAT_OUTPUT_FLAG_PCT = STAT +POINT_STAT_OUTPUT_FLAG_PSTD = STAT +POINT_STAT_OUTPUT_FLAG_PJC = STAT +POINT_STAT_OUTPUT_FLAG_PRC = STAT +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = +#POINT_STAT_OUTPUT_FLAG_ORANK = + +POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + +#POINT_STAT_HSS_EC_VALUE = + +# Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be +# considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file +OBS_POINT_STAT_WINDOW_BEGIN = {OBS_WINDOW_BEGIN} +OBS_POINT_STAT_WINDOW_END = {OBS_WINDOW_END} + +# Optional list of offsets to look for point observation data +POINT_STAT_OFFSETS = 0 # Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. MODEL = {ENV[MODEL]}_prob + +POINT_STAT_DESC = NA OBTYPE = NDAS # Regrid to specified grid. Indicate NONE if no regridding, or the grid id +# (e.g. G212) POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 POINT_STAT_OUTPUT_PREFIX = {ENV[MODEL]}_ADPUPA_{OBTYPE}_prob +# sets the -obs_valid_beg command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_BEG = {valid?fmt=%Y%m%d_%H} + +# sets the -obs_valid_end command line argument (optional) +# not used for this example +#POINT_STAT_OBS_VALID_END = {valid?fmt=%Y%m%d_%H} + # Verification Masking regions # Indicate which grid and polygon masking region, if applicable -POINT_STAT_GRID = +POINT_STAT_GRID = # List of full path to poly masking files. NOTE: Only short lists of poly # files work (those that fit on one line), a long list will result in an @@ -123,10 +175,17 @@ POINT_STAT_STATION_ID = # Message types, if all message types are to be returned, leave this empty, # otherwise indicate the message types of interest. POINT_STAT_MESSAGE_TYPE = ADPUPA - # Variables and levels as specified in the field dictionary of the MET # point_stat configuration file. Specify as FCST_VARn_NAME, FCST_VARn_LEVELS, # (optional) FCST_VARn_OPTION + +# set to True to run PointStat once for each name/level combination +# set to False to run PointStat once per run time including all fields +POINT_STAT_ONCE_PER_FIELD = False + +# fields to compare +# Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set +# To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_P850_ENS_FREQ_ge288 FCST_VAR1_LEVELS = (*,*) FCST_VAR1_THRESH = ==0.1 @@ -449,3 +508,64 @@ OBS_VAR39_THRESH = >1500 # Forecast data description variables FCST_IS_PROB = True FCST_PROB_IN_GRIB_PDS = False + +# End of [config] section and start of [dir] section +[dir] + +# directory containing input to PB2NC +PB2NC_INPUT_DIR = {ENV[OBS_DIR]} + +# directory to write output from +PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc + +INPUT_BASE = {ENV[INPUT_BASE]} + +FCST_POINT_STAT_INPUT_DIR = {INPUT_BASE} +OBS_POINT_STAT_INPUT_DIR = {PB2NC_OUTPUT_DIR} + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_DIR = + +# directory containing climatology mean input to PointStat +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_DIR = + +OUTPUT_BASE = {ENV[EXPTDIR]} +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} + +LOG_DIR = {ENV[EXPTDIR]}/log + +STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_prob + +# End of [dir] section and start of [filename_templates] section +[filename_templates] + +# Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR +PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} + +# Template to use to write output from PB2NC +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR +FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc + +# Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = + +# Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_STDEV_INPUT_DIR +# Not used in this example +POINT_STAT_CLIMO_STDEV_INPUT_TEMPLATE = + +#Template for where point-stat output is written +POINT_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/metprd/ensemble_stat_prob + +# Specify the name of the metplus.log file +LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa + +# Specify where the location and name of the final metplus_final.conf +METPLUS_CONF={OUTPUT_BASE}/{ENV[CDATE]}/metprd/ensemble_stat_prob/metplus_final.upper_air.conf diff --git a/ush/templates/parm/metplus/REFC.conf b/ush/templates/parm/metplus/REFC.conf deleted file mode 100644 index a661fabc89..0000000000 --- a/ush/templates/parm/metplus/REFC.conf +++ /dev/null @@ -1,102 +0,0 @@ -# Composite Reflectivity Verification Options - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} - -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} - -STAGING_DIR = {OUTPUT_BASE}/stage/REFC - -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.REFC.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = MRMS -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = GridStat - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# Forecast/Observation variable Information -FCST_VAR1_NAME = REFC -FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = cnt_thresh = [ >15 ]; cnt_logic = UNION; - -BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 - -OBS_VAR1_NAME = MergedReflectivityQCComposite -OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = censor_thresh = [eq-999, <-20]; censor_val = [-9999, -20]; cnt_thresh = [ >15 ]; cnt_logic = UNION; - -OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 -OBS_GRID_STAT_FILE_WINDOW_END = 300 - -# Neighborhood shape and width -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 1, 3, 5, 7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/RETOP.conf b/ush/templates/parm/metplus/RETOP.conf deleted file mode 100644 index c09ea7a18d..0000000000 --- a/ush/templates/parm/metplus/RETOP.conf +++ /dev/null @@ -1,102 +0,0 @@ -# Echo Top Verification Options - -[dir] -# Input and output data directories -INPUT_BASE = {ENV[INPUT_BASE]} -FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} - -OUTPUT_BASE = {ENV[OUTPUT_BASE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} - -LOG_DIR = {ENV[EXPTDIR]}/log - -OBS_GRID_STAT_INPUT_DIR = {ENV[OBS_DIR]} - -STAGING_DIR = {OUTPUT_BASE}/stage/RETOP - -[filename_templates] -# format of filenames -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 - -# ANLYS -OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 - -GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat - -# Location of MET poly files -GRID_STAT_VERIFICATION_MASK_TEMPLATE = {MET_INSTALL_DIR}/share/met/poly/CONUS.poly - -# Specify the name of the metplus.log file -LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} - -# Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.RETOP.conf - -[config] -# Model to verify -MODEL = {ENV[MODEL]} -FCST_NATIVE_DATA_TYPE = GRIB - -# Set obtype to vx -OBTYPE = MRMS -OBS_NATIVE_DATA_TYPE = GRIB - -# List of applications to run -PROCESS_LIST = GridStat - -# location of grid_stat MET config file -GRID_STAT_CONFIG_FILE = {CONFIG_DIR}/GridStatConfig_REFC -GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_{OBTYPE} - -# time looping - options are INIT, VALID, RETRO, and REALTIME -LOOP_BY = INIT - -# Format of INIT_BEG and INIT_END -INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run -INIT_BEG = {ENV[CDATE]} -# End time for METplus run -INIT_END = {ENV[CDATE]} -# Increment between METplus runs in seconds. Must be > 60 -INIT_INCREMENT = 3600 - -# list of forecast leads to process. -#LEAD_SEQ = begin_end_incr(1,24,1) -LEAD_SEQ = {ENV[fhr_list]} - -# Options are times, processes -# times = run all items in the PROCESS_LIST for a single initialization -# time, then repeat until all times have been evaluated. -# processes = run each item in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST. -LOOP_ORDER = times - -# run pcp_combine on forecast/obs data? -FCST_PCP_COMBINE_RUN = False -OBS_PCP_COMBINE_RUN = False - -# Set grid to verify on -GRID_STAT_REGRID_TO_GRID = FCST - -# Forecast/Observation variable Information -FCST_VAR1_NAME = RETOP -FCST_VAR1_LEVELS = L0 -FCST_VAR1_OPTIONS = convert(x) = M_to_KFT(x); cnt_thresh = [ >0 ]; cnt_logic = UNION; - -BOTH_VAR1_THRESH = ge20, ge30, ge40, ge50 - -OBS_VAR1_NAME = EchoTop18 -OBS_VAR1_LEVELS = Z500 -OBS_VAR1_OPTIONS = convert(x) = KM_to_KFT(x); censor_thresh = [<=-9.84252,eq-3.28084]; censor_val = [-9999,-16.4042]; cnt_thresh = [ >0 ]; cnt_logic = UNION; - -OBS_GRID_STAT_FILE_WINDOW_BEGIN = -300 -OBS_GRID_STAT_FILE_WINDOW_END = 300 - -# Neighborhood shape and width -GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE -GRID_STAT_NEIGHBORHOOD_WIDTH = 1, 3, 5, 7 - -# Forecast data description variables -FCST_IS_PROB = False - diff --git a/ush/templates/parm/metplus/common.conf b/ush/templates/parm/metplus/common.conf index a04573febb..01cc667460 100644 --- a/ush/templates/parm/metplus/common.conf +++ b/ush/templates/parm/metplus/common.conf @@ -12,7 +12,7 @@ MET_BIN_DIR = {MET_INSTALL_DIR}/{ENV[MET_BIN_EXEC]} METPLUS_PARM_BASE = {ENV[METPLUS_CONF]} # Location of configuration files used by MET applications -CONFIG_DIR = {ENV[MET_CONFIG]} +CONFIG_DIR = {METPLUS_BASE}/parm/met_config # Other directories TMP_DIR = /tmp From d03d311f785657539543ce117d23bdc3794ffdfc Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 23 Mar 2022 14:14:47 -0600 Subject: [PATCH 138/203] Add missing user-defined stochastic physics options; fix stochastic physics seed generation script (#704) ## DESCRIPTION OF CHANGES: Add missing user-defined options for tendency-based stochastic physics and fix the ensemble-based seed generation script to work regardless of whether stochastic physics is turned on or not. ## TESTS CONDUCTED: Tested on Hera using the following WE2E configurations with and without stochastic physics: config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh config.community_ensemble_2mems.sh ## ISSUE (optional): [Issue #702](https://github.com/ufs-community/regional_workflow/issues/702) ## CONTRIBUTORS (optional): Thanks to @mkavulich and @chan-hoo for finding this problem. --- scripts/exregional_run_fcst.sh | 7 +- ush/config_defaults.sh | 9 +++ ush/generate_FV3LAM_wflow.sh | 64 ++++++++++----- ...arams.sh => set_FV3nml_ens_stoch_seeds.sh} | 81 +++++++++++-------- 4 files changed, 104 insertions(+), 57 deletions(-) rename ush/{set_FV3nml_stoch_params.sh => set_FV3nml_ens_stoch_seeds.sh} (87%) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index d7c3d48f62..331bb6ab49 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -9,7 +9,7 @@ # . ${GLOBAL_VAR_DEFNS_FP} . $USHDIR/source_util_funcs.sh -. $USHDIR/set_FV3nml_stoch_params.sh +. $USHDIR/set_FV3nml_ens_stoch_seeds.sh # #----------------------------------------------------------------------- # @@ -444,8 +444,9 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . fi -if [ "${DO_ENSEMBLE}" = TRUE ]; then - set_FV3nml_stoch_params cdate="$cdate" || print_err_msg_exit "\ +if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] \ + [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then + set_FV3nml_ens_stoch_seeds cdate="$cdate" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 4c7b4d7e8c..466344f682 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1504,6 +1504,7 @@ FIXgsm_FILES_TO_COPY_TO_FIXam=( \ "global_hyblev.l65.txt" \ "global_zorclim.1x1.grb" \ "global_sfc_emissivity_idx.txt" \ +"global_tg3clim.2.6x1.5.grb" \ "global_solarconstant_noaa_an.txt" \ "global_albedo4.1x1.grb" \ "geo_em.d01.lat-lon.2.5m.HGT_M.nc" \ @@ -1558,6 +1559,7 @@ CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING=( \ "global_h2oprdlos.f77 | global_h2o_pltc.f77" \ "global_albedo4.1x1.grb | global_albedo4.1x1.grb" \ "global_zorclim.1x1.grb | global_zorclim.1x1.grb" \ +"global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb" \ "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt" \ "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt" \ "global_o3prdlos.f77 | " \ @@ -1793,18 +1795,25 @@ NUM_ENS_MEMBERS="1" DO_SHUM="FALSE" DO_SPPT="FALSE" DO_SKEB="FALSE" +ISEED_SPPT="1" +ISEED_SHUM="2" +ISEED_SKEB="3" +NEW_LSCALE="TRUE" SHUM_MAG="0.006" #Variable "shum" in input.nml SHUM_LSCALE="150000" SHUM_TSCALE="21600" #Variable "shum_tau" in input.nml SHUM_INT="3600" #Variable "shumint" in input.nml SPPT_MAG="0.7" #Variable "sppt" in input.nml +SPPT_LOGIT="TRUE" SPPT_LSCALE="150000" SPPT_TSCALE="21600" #Variable "sppt_tau" in input.nml SPPT_INT="3600" #Variable "spptint" in input.nml +SPPT_SFCLIMIT="TRUE" SKEB_MAG="0.5" #Variable "skeb" in input.nml SKEB_LSCALE="150000" SKEB_TSCALE="21600" #Variable "skeb_tau" in input.nml SKEB_INT="3600" #Variable "skebint" in input.nml +SKEBNORM="1" SKEB_VDOF="10" USE_ZMTNBLCK="FALSE" # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 7c840a955f..88c6493a83 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -790,22 +790,6 @@ settings="\ 'lndp_type': ${LNDP_TYPE}, 'lndp_each_step': ${LSM_SPP_EACH_STEP}, 'fhcyc': ${FHCYC_LSM_SPP_OR_NOT}, - } -'nam_stochy': { - 'shum': ${SHUM_MAG}, - 'shum_lscale': ${SHUM_LSCALE}, - 'shum_tau': ${SHUM_TSCALE}, - 'shumint': ${SHUM_INT}, - 'sppt': ${SPPT_MAG}, - 'sppt_lscale': ${SPPT_LSCALE}, - 'sppt_tau': ${SPPT_TSCALE}, - 'spptint': ${SPPT_INT}, - 'skeb': ${SKEB_MAG}, - 'skeb_lscale': ${SKEB_LSCALE}, - 'skeb_tau': ${SKEB_TSCALE}, - 'skebint': ${SKEB_INT}, - 'skeb_vdof': ${SKEB_VDOF}, - 'use_zmtnblck': ${USE_ZMTNBLCK}, }" # # Add to "settings" the values of those namelist variables that specify @@ -862,6 +846,51 @@ done settings="$settings }" # +# Add the relevant tendency-based stochastic physics namelist variables to +# "settings" when running with SPPT, SHUM, or SKEB turned on. Otherwise +# only include an empty "nam_stochy" stanza. +# +settings="$settings +'nam_stochy': {" +if [ "${DO_SPPT}" = "TRUE" ]; then + settings="$settings + 'iseed_sppt': ${ISEED_SPPT}, + 'new_lscale': ${NEW_LSCALE}, + 'sppt': ${SPPT_MAG}, + 'sppt_logit': ${SPPT_LOGIT}, + 'sppt_lscale': ${SPPT_LSCALE}, + 'sppt_sfclimit': ${SPPT_SFCLIMIT}, + 'sppt_tau': ${SPPT_TSCALE}, + 'spptint': ${SPPT_INT}, + 'use_zmtnblck': ${USE_ZMTNBLCK}," +fi + +if [ "${DO_SHUM}" = "TRUE" ]; then + settings="$settings + 'iseed_shum': ${ISEED_SHUM}, + 'new_lscale': ${NEW_LSCALE}, + 'shum': ${SHUM_MAG}, + 'shum_lscale': ${SHUM_LSCALE}, + 'shum_tau': ${SHUM_TSCALE}, + 'shumint': ${SHUM_INT}, + 'use_zmtnblck': ${USE_ZMTNBLCK}," +fi + +if [ "${DO_SKEB}" = "TRUE" ]; then + settings="$settings + 'iseed_skeb': ${ISEED_SKEB}, + 'new_lscale': ${NEW_LSCALE}, + 'skeb': ${SKEB_MAG}, + 'skeb_lscale': ${SKEB_LSCALE}, + 'skebnorm': ${SKEBNORM}, + 'skeb_tau': ${SKEB_TSCALE}, + 'skebint': ${SKEB_INT}, + 'skeb_vdof': ${SKEB_VDOF}, + 'use_zmtnblck': ${USE_ZMTNBLCK}," +fi +settings="$settings + }" +# # Add the relevant SPP namelist variables to "settings" when running with # SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. # @@ -1067,9 +1096,6 @@ fi } - - - # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_ens_stoch_seeds.sh similarity index 87% rename from ush/set_FV3nml_stoch_params.sh rename to ush/set_FV3nml_ens_stoch_seeds.sh index 8abfc45d47..64322d6648 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_ens_stoch_seeds.sh @@ -13,7 +13,7 @@ # #----------------------------------------------------------------------- # -function set_FV3nml_stoch_params() { +function set_FV3nml_ens_stoch_seeds() { # #----------------------------------------------------------------------- # @@ -98,48 +98,67 @@ function set_FV3nml_stoch_params() { ensmem_num=$((10#${ENSMEM_INDX})) + settings="\ +'nam_stochy': {" + + if [ ${DO_SPPT} = TRUE ]; then + + iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) + settings="$settings + 'iseed_sppt': ${iseed_sppt}," + + fi + + if [ ${DO_SHUM} = TRUE ]; then + iseed_shum=$(( cdate*1000 + ensmem_num*10 + 2 )) + settings="$settings + 'iseed_shum': ${iseed_shum}," + + fi + + if [ ${DO_SKEB} = TRUE ]; then + iseed_skeb=$(( cdate*1000 + ensmem_num*10 + 3 )) - iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) - iseed_lsm_spp=$(( cdate*1000 + ensmem_num*10 + 9)) + settings="$settings + 'iseed_skeb': ${iseed_skeb}," + + fi + settings="$settings + }" + + settings="$settings +'nam_sppperts': {" + + if [ ${DO_SPP} = TRUE ]; then num_iseed_spp=${#ISEED_SPP[@]} for (( i=0; i<${num_iseed_spp}; i++ )); do iseed_spp[$i]=$(( cdate*1000 + ensmem_num*10 + ${ISEED_SPP[$i]} )) done - settings="" - - if [ ${DO_SPPT} = TRUE ] || [ ${DO_SHUM} = TRUE ] || [ ${DO_SKEB} = TRUE ]; then - - settings=$settings"\ -'nam_stochy': { - 'iseed_shum': ${iseed_shum}, - 'iseed_skeb': ${iseed_skeb}, - 'iseed_sppt': ${iseed_sppt}, - } -" + settings="$settings + 'iseed_spp': [ $( printf "%s, " "${iseed_spp[@]}" ) ]," + fi - if [ ${DO_SPP} = TRUE ]; then + settings="$settings + }" - settings=$settings"\ -'nam_sppperts': { - 'iseed_spp': [ $( printf "%s, " "${iseed_spp[@]}" ) ] - } -" - fi + settings="$settings +'nam_sfcperts': {" if [ ${DO_LSM_SPP} = TRUE ]; then + + iseed_lsm_spp=$(( cdate*1000 + ensmem_num*10 + 9)) + + settings="$settings + 'iseed_lndp': [ $( printf "%s, " "${iseed_lsm_spp[@]}" ) ]," - settings=$settings"\ -'nam_sfcperts': { - 'iseed_lndp': [ $( printf "%s, " "${iseed_lsm_spp[@]}" ) ] - } -" fi - if [ -n "$settings" ]; then + settings="$settings + }" $USHDIR/set_namelist.py -q \ -n ${FV3_NML_FP} \ @@ -156,14 +175,6 @@ failed. Parameters passed to this script are: Namelist settings specified on command line (these have highest precedence): settings = $settings" - - else - - print_info_msg "\ -The variable \"settings\" is empty, so not setting any namelist values." - - fi - # #----------------------------------------------------------------------- # From 20df567b4ee8ca2b7db290f61f20349139512c30 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Wed, 23 Mar 2022 15:23:16 -0600 Subject: [PATCH 139/203] Add namelist option for netCDF4 when running with the 3-km NA domain; update NAM HPSS settings and WE2E tests (#707) * Change to netcdf4 when using the NA 3-km domain * Update HPSS paths for NAM data * Update NAM HPSS locations and dates for WE2E tests. * Remove lines from merge. --- ...rid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh | 8 ++++---- ...S_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh | 8 ++++---- ush/generate_FV3LAM_wflow.sh | 9 +++++++++ ush/get_extrn_mdl_file_dir_info.sh | 11 +++++++---- ush/templates/data_locations.yml | 6 +++--- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh index a91c10151a..10be9fed96 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh @@ -17,9 +17,9 @@ EXTRN_MDL_NAME_ICS="NAM" EXTRN_MDL_NAME_LBCS="NAM" USE_USER_STAGED_EXTRN_FILES="TRUE" -DATE_FIRST_CYCL="20150602" -DATE_LAST_CYCL="20150602" -CYCL_HRS=( "12" ) +DATE_FIRST_CYCL="20210615" +DATE_LAST_CYCL="20210615" +CYCL_HRS=( "00" ) -FCST_LEN_HRS="24" +FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh index a22466e5b4..540e5a0589 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh @@ -17,9 +17,9 @@ EXTRN_MDL_NAME_ICS="NAM" EXTRN_MDL_NAME_LBCS="NAM" USE_USER_STAGED_EXTRN_FILES="TRUE" -DATE_FIRST_CYCL="20150602" -DATE_LAST_CYCL="20150602" -CYCL_HRS=( "12" ) +DATE_FIRST_CYCL="20210615" +DATE_LAST_CYCL="20210615" +CYCL_HRS=( "00" ) -FCST_LEN_HRS="24" +FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 88c6493a83..d0f5d3bd4b 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -846,6 +846,15 @@ done settings="$settings }" # +# Use netCDF4 when running the North American 3-km domain due to file size. +# +if [ "${PREDEF_GRID_NAME}" = "RRFS_NA_3km" ]; then +settings="$settings +'fms2_io_nml': { + 'netcdf_default_format': netcdf4, + }" +fi +# # Add the relevant tendency-based stochastic physics namelist variables to # "settings" when running with SPPT, SHUM, or SKEB turned on. Otherwise # only include an empty "nam_stochy" stanza. diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 30a6e2d887..aab64c2dc1 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -312,7 +312,7 @@ function get_extrn_mdl_file_dir_info() { "NAM") fns=( "" ) - fns_in_arcv=( "nam.t${hh}z.bgrdsf${fcst_hh}.tm00" ) + fns_in_arcv=( "nam.t${hh}z.awphys${fcst_hh}.tm00.grib2" ) ;; *) @@ -377,7 +377,7 @@ and analysis or forecast (anl_or_fcst): ;; "NAM") - fn_tmpl="nam.t${hh}z.bgrdsfFHR3" + fn_tmpl="nam.t${hh}z.awphysFHR2.tm00.grib2" ;; *) @@ -599,11 +599,14 @@ bination of external model (extrn_mdl_name) and analysis or forecast ;; "NAM") + # + # 12-km CONUS 218 grid. + # arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" arcv_fmt="tar" - arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.bgrid.${arcv_fmt}" + arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.awphys.${arcv_fmt}" arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" + arcvrel_dir="." ;; *) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index 23354e1719..c3efd9a29d 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -184,10 +184,10 @@ NAM: archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_file_names: - - com_nam_prod_nam.{yyyymmddhh}.bgrid.tar + - com_nam_prod_nam.{yyyymmddhh}.awphys{fcst_hr:02d}.tar file_names: anl: - - nam.t{hh}z.bgrdsf{fcst_hr:03d}.tm00 + - nam.t{hh}z.awphys{fcst_hr:02d}.tm00.grib2 fcst: - - nam.t{hh}z.bgrdsf{fcst_hr:03d} + - nam.t{hh}z.awphys{fcst_hr:02d}.tm00.grib2 From d6913a2bf8a262b876f40b9131a86021fce338d3 Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Wed, 23 Mar 2022 17:46:04 -0400 Subject: [PATCH 140/203] Tweaks to allow compiler and build_env_fn to be specified in the run_WE2E_test.sh script (#711) --- tests/WE2E/run_WE2E_tests.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index bfab3128a5..4ee8b504ed 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -92,7 +92,9 @@ Usage: [verbose=\"...\"] \\ [machine_file=\"...\"] \\ [stmp=\"...\"] \\ - [ptmp=\"...\"] + [ptmp=\"...\"] \\ + [compiler=\"...\"] \\ + [build_env_fn=\"...\"] The arguments in brackets are optional. The arguments are defined as follows: @@ -204,6 +206,16 @@ argument is not used for any tests that are not in NCO mode. ptmp: Same as the argument \"stmp\" described above but for setting the experiment variable PTMP for all tests that will run in NCO mode. + +compiler: +Type of compiler to use for the workflow. Options are \"intel\" +and \"gnu\". Default is \"intel\", + +build_env_fn: +Specify the build environment (see ufs-srweather-app/envs) to +use for the workflow. (e.g. build_cheyenne_gnu.env). If a +\"gnu\" compiler is specified, it must also be specified with +the \"compiler\" option. " # #----------------------------------------------------------------------- @@ -240,6 +252,8 @@ valid_args=( \ "machine_file" \ "stmp" \ "ptmp" \ + "compiler" \ + "build_env_fn" \ ) process_args valid_args "$@" # @@ -670,7 +684,8 @@ Please correct and rerun." # MACHINE="${machine^^}" ACCOUNT="${account}" - + COMPILER=${compiler:-"intel"} + BUILD_ENV_FN=${build_env_fn:-"build_${machine}_${COMPILER}.env"} EXPT_BASEDIR="${expt_basedir}" EXPT_SUBDIR="${test_name}" USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} @@ -692,7 +707,10 @@ Please correct and rerun." # subdirectory. # MACHINE=\"${MACHINE}\" -ACCOUNT=\"${ACCOUNT}\"" +ACCOUNT=\"${ACCOUNT}\" + +COMPILER=\"${COMPILER}\" +BUILD_ENV_FN=\"${BUILD_ENV_FN}\"" if [ -n "${exec_subdir}" ]; then expt_config_str=${expt_config_str}" From 09236d09ace9f09a6678cc25d8ec6f5eed76cc28 Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Wed, 23 Mar 2022 16:10:23 -0600 Subject: [PATCH 141/203] Changed 20200304 to 20200303 in ush/mrms_pull_topofhour.py (#712) --- ush/mrms_pull_topofhour.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py index 43bc706193..bd98b28058 100644 --- a/ush/mrms_pull_topofhour.py +++ b/ush/mrms_pull_topofhour.py @@ -48,9 +48,9 @@ # Sort list of files for each MRMS product print(valid.strftime('%Y%m%d')) -if valid.strftime('%Y%m%d') < '20200304': +if valid.strftime('%Y%m%d') < '20200303': search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' -elif valid.strftime('%Y%m%d') >= '20200304': +elif valid.strftime('%Y%m%d') >= '20200303': search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' file_list = [f for f in glob.glob(search_path)] time_list = [file_list[x][-24:-9] for x in range(len(file_list))] @@ -69,12 +69,12 @@ filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz' filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz' - if valid.strftime('%Y%m%d') < '20200304': + if valid.strftime('%Y%m%d') < '20200303': print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) os.system('gunzip '+VALID_DIR+'/'+filename2) - elif valid.strftime('%Y%m%d') >= '20200304': + elif valid.strftime('%Y%m%d') >= '20200303': print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) From 8d7e64ffaadbacdb82bde49d5c1d073154e9d2ea Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Thu, 31 Mar 2022 09:55:43 -0600 Subject: [PATCH 142/203] Remove unused rocoto directory in ush (#720) --- ush/rocoto/.gitignore | 47 - ush/rocoto/fv3gfs_workflow.sh | 52 - ush/rocoto/rocoto.py | 346 ---- ush/rocoto/rocoto_viewer.py | 2383 ------------------------- ush/rocoto/setup_expt.py | 199 --- ush/rocoto/setup_expt_fcstonly.py | 159 -- ush/rocoto/setup_workflow.py | 831 --------- ush/rocoto/setup_workflow_fcstonly.py | 378 ---- ush/rocoto/workflow_utils.py | 341 ---- 9 files changed, 4736 deletions(-) delete mode 100644 ush/rocoto/.gitignore delete mode 100755 ush/rocoto/fv3gfs_workflow.sh delete mode 100755 ush/rocoto/rocoto.py delete mode 100755 ush/rocoto/rocoto_viewer.py delete mode 100755 ush/rocoto/setup_expt.py delete mode 100755 ush/rocoto/setup_expt_fcstonly.py delete mode 100755 ush/rocoto/setup_workflow.py delete mode 100755 ush/rocoto/setup_workflow_fcstonly.py delete mode 100755 ush/rocoto/workflow_utils.py diff --git a/ush/rocoto/.gitignore b/ush/rocoto/.gitignore deleted file mode 100644 index 7f98f9b45c..0000000000 --- a/ush/rocoto/.gitignore +++ /dev/null @@ -1,47 +0,0 @@ -# Compiled source # -################### -*.com -*.class -*.dll -*.exe -*.o -*.mod -*.so -*.pyc - -# Temporary files # -################### -*.swp -*.swo -*~ - -# Packages # -############ -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip - -# Logs and databases # -###################### -*.log -*.sql -*.sqlite - -# OS generated files # -###################### -.DS_Store* -ehthumbs.db -Icon? -Thumbs.db - -*.lock -package-lock.json - -# Subversion and Git directories -.svn -.git diff --git a/ush/rocoto/fv3gfs_workflow.sh b/ush/rocoto/fv3gfs_workflow.sh deleted file mode 100755 index 7e52c6e760..0000000000 --- a/ush/rocoto/fv3gfs_workflow.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh - -# Checkout, build, setup and execute the workflow - -set -ex - -fv3gfs_tag="https://svnemc.ncep.noaa.gov/projects/fv3gfs/trunk" - -pslot="fv3test" -expdir="/path/to/expdir" -comrot="/path/to/comrot" -fv3gfs="/path/to/fv3gfs_tag/checkout" -idate="2017073118" -edate="2017080112" - -###################################### -# USER NEED NOT MODIFY BELOW THIS LINE -###################################### - -if [ -d /scratch4/NCEPDEV ]; then - machine="theia" - icsdir="/scratch4/NCEPDEV/global/noscrub/glopara/ICS/FV3GFS" -elif [ -d /gpfs/hps3 ]; then - machine="cray" - icsdir="/gpfs/hps3/emc/global/noscrub/emc.glopara/ICS" -else - echo "Unknown machine $machine, ABORT!" - exit -1 -fi - -[[ -d $expdir/$pslot ]] && rm -rf $expdir/$pslot -[[ -d $comrot/$pslot ]] && rm -rf $comrot/$pslot -[[ -d $fv3gfs/$pslot ]] && rm -rf $fv3gfs/$pslot - -gfs_ver=v15.0.0 -mkdir -p $fv3gfs -cd $fv3gfs -git clone --recursive gerrit:fv3gfs gfs.${gfs_ver} - -cd $fv3gfs/gfs.${gfs_ver}/sorc -sh checkout.sh -sh build_all.sh $machine -sh link_fv3gfs.sh emc $machine - -cd $fv3gfs/gfs.${gfs_ver}/ush/rocoto -python setup_expt.py --pslot $pslot --comrot $comrot --expdir $expdir --idate $idate --edate $edate --icsdir $icsdir --configdir ../parm/config -python setup_workflow.py --expdir $expdir/$pslot - -cd $expdir/$pslot -crontab $pslot.crontab - -exit diff --git a/ush/rocoto/rocoto.py b/ush/rocoto/rocoto.py deleted file mode 100755 index 0a0fdf7d86..0000000000 --- a/ush/rocoto/rocoto.py +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### -''' - MODULE: - rocoto.py - - ABOUT: - Helper module to create tasks, metatasks, and dependencies - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov -''' - -def create_metatask(task_dict, metatask_dict): - ''' - create a Rocoto metatask given a dictionary containing task and metatask information - :param metatask_dict: metatask key-value parameters - :type metatask_dict: dict - :param task_dict: task key-value parameters - :type task_dict: dict - :return: Rocoto metatask - :rtype: list - ''' - - # Grab metatask info from the metatask_dict - metataskname = metatask_dict.get('metataskname', 'demometatask') - varname = metatask_dict.get('varname', 'demovar') - varval = metatask_dict.get('varval', 1) - vardict = metatask_dict.get('vardict', None) - - strings = [] - - strings.append('\n' % metataskname) - strings.append('\n') - strings.append('\t%s\n' % (varname, str(varval))) - if vardict is not None: - for key in vardict.keys(): - value = str(vardict[key]) - strings.append('\t%s\n' % (key, value)) - strings.append('\n') - tasklines = create_task(task_dict) - for tl in tasklines: - strings.append('%s' % tl) if tl == '\n' else strings.append('\t%s' % tl) - strings.append('\n') - strings.append('\n') - - return strings - - -def create_task(task_dict): - ''' - create a Rocoto task given a dictionary containing task information - :param task_dict: task key-value parameters - :type task_dict: dict - :return: Rocoto task - :rtype: list - ''' - - # Grab task info from the task_dict - taskname = task_dict.get('taskname', 'demotask') - cycledef = task_dict.get('cycledef', 'democycle') - maxtries = task_dict.get('maxtries', 3) - final = task_dict.get('final', False) - command = task_dict.get('command', 'sleep 10') - jobname = task_dict.get('jobname', 'demojob') - account = task_dict.get('account', 'batch') - queue = task_dict.get('queue', 'debug') - walltime = task_dict.get('walltime', '00:01:00') - log = task_dict.get('log', 'demo.log') - native = task_dict.get('native', None) - memory = task_dict.get('memory', None) - resources = task_dict.get('resources', None) - envar = task_dict.get('envar', None) - dependency = task_dict.get('dependency', None) - - str_maxtries = str(maxtries) - str_final = ' final="true"' if final else '' - envar = envar if isinstance(envar, list) else [envar] - - strings = [] - - strings.append('\n' % \ - (taskname, cycledef, str_maxtries, str_final)) - strings.append('\n') - strings.append('\t%s\n' % command) - strings.append('\n') - strings.append('\t%s\n' % jobname) - strings.append('\t%s\n' % account) - strings.append('\t%s\n' % queue) - if resources is not None: - strings.append('\t%s\n' % resources) - strings.append('\t%s\n' % walltime) - if memory is not None: - strings.append('\t%s\n' % memory) - if native is not None: - strings.append('\t%s\n' % native) - strings.append('\n') - strings.append('\t%s\n' % log) - strings.append('\n') - - if envar[0] is not None: - for e in envar: - strings.append('\t%s\n' % e) - strings.append('\n') - - if dependency is not None: - strings.append('\t\n') - for d in dependency: - strings.append('\t\t%s\n' % d) - strings.append('\t\n') - strings.append('\n') - - strings.append('\n') - - return strings - - -def add_dependency(dep_dict): - ''' - create a simple Rocoto dependency given a dictionary with dependency information - :param dep_dict: dependency key-value parameters - :type dep_dict: dict - :return: Rocoto simple dependency - :rtype: str - ''' - - dep_condition = dep_dict.get('condition', None) - dep_type = dep_dict.get('type', None) - - if dep_type in ['task', 'metatask']: - - string = add_task_tag(dep_dict) - - elif dep_type in ['data']: - - string = add_data_tag(dep_dict) - - elif dep_type in ['cycleexist']: - - string = add_cycle_tag(dep_dict) - - elif dep_type in ['streq', 'strneq']: - - string = add_streq_tag(dep_dict) - - else: - - msg = 'Unknown dependency type %s' % dep_dict['type'] - raise KeyError(msg) - - if dep_condition is not None: - string = '<%s>%s' % (dep_condition, string, dep_condition) - - return string - - -def add_task_tag(dep_dict): - ''' - create a simple task or metatask tag - :param dep_dict: dependency key-value parameters - :type dep_dict: dict - :return: Rocoto simple task or metatask dependency - :rtype: str - ''' - - dep_type = dep_dict.get('type', None) - dep_name = dep_dict.get('name', None) - dep_offset = dep_dict.get('offset', None) - - if dep_name is None: - msg = 'a %s name is necessary for %s dependency' % (dep_type, dep_type) - raise KeyError(msg) - - string = '<' - string += '%sdep %s="%s"' % (dep_type, dep_type, dep_name) - if dep_offset is not None: - string += ' cycle_offset="%s"' % dep_offset - string += '/>' - - return string - -def add_data_tag(dep_dict): - ''' - create a simple data tag - :param dep_dict: dependency key-value parameters - :type dep_dict: dict - :return: Rocoto simple task or metatask dependency - :rtype: str - ''' - - dep_type = dep_dict.get('type', None) - dep_data = dep_dict.get('data', None) - dep_offset = dep_dict.get('offset', None) - - if dep_data is None: - msg = 'a data value is necessary for %s dependency' % dep_type - raise KeyError(msg) - - if dep_offset is None: - if '@' in dep_data: - offset_string_b = '' - offset_string_e = '' - else: - offset_string_b = '' - offset_string_e = '' - else: - offset_string_b = '' % dep_offset - offset_string_e = '' - - string = '' - string += '%s%s%s' % (offset_string_b, dep_data, offset_string_e) - string += '' - - return string - -def add_cycle_tag(dep_dict): - ''' - create a simple cycle exist tag - :param dep_dict: dependency key-value parameters - :type dep_dict: dict - :return: Rocoto simple task or metatask dependency - :rtype: str - ''' - - dep_type = dep_dict.get('type', None) - dep_offset = dep_dict.get('offset', None) - - if dep_offset is None: - msg = 'an offset value is necessary for %s dependency' % dep_type - raise KeyError(msg) - - string = '' % dep_offset - - return string - -def add_streq_tag(dep_dict): - ''' - create a simple string comparison tag - :param dep_dict: dependency key-value parameters - :type dep_dict: dict - :return: Rocoto simple task or metatask dependency - :rtype: str - ''' - - dep_type = dep_dict.get('type', None) - dep_left = dep_dict.get('left', None) - dep_right = dep_dict.get('right', None) - - fail = False - msg = '' - if dep_left is None: - msg += 'a left value is necessary for %s dependency' % dep_type - fail = True - if dep_right is None: - if fail: - msg += '\n' - msg += 'a right value is necessary for %s dependency' % dep_type - fail = True - if fail: - raise KeyError(msg) - - string = '<%s>%s%s' % (dep_type, dep_left, dep_right, dep_type) - - return string - - -def _traverse(o, tree_types=(list, tuple)): - ''' - Traverse through a list of lists or tuples and yeild the value - Objective is to flatten a list of lists or tuples - :param o: list of lists or not - :type o: list, tuple, scalar - :param tree_types: trees to travers - :type tree_types: tuple - :return: value in the list or tuple - :rtype: scalar - ''' - - if isinstance(o, tree_types): - for value in o: - for subvalue in _traverse(value, tree_types): - yield subvalue - else: - yield o - - -def create_dependency(dep_condition=None, dep=None): - ''' - create a compound dependency given a list of dependendies, and compounding condition - the list of dependencies are created using add_dependency - :param dep_condition: dependency condition - :type dep_condition: boolean e.g. and, or, true, false - :param dep: dependency - :type dep: str or list - :return: Rocoto compound dependency - :rtype: list - ''' - - dep = dep if isinstance(dep, list) else [dep] - - strings = [] - - if dep_condition is not None: - strings.append('<%s>' % dep_condition) - - if dep[0] is not None: - for d in dep: - if dep_condition is None: - strings.append('%s' % d) - else: - for e in _traverse(d): - strings.append('\t%s' % e) - - if dep_condition is not None: - strings.append('' % dep_condition) - - return strings - - -def create_envar(name=None,value=None): - ''' - create an Rocoto environment variable given name and value - returns the environment variable as a string - :param name: name of the environment variable - :type name: str - :param value: value of the environment variable - :type value: str or float or int or unicode - :return: Rocoto environment variable key-value pair - :rtype: str - ''' - - string = '' - string += '' - string += '%s' % name - string += '%s' % str(value) - string += '' - - return string diff --git a/ush/rocoto/rocoto_viewer.py b/ush/rocoto/rocoto_viewer.py deleted file mode 100755 index 26db6943fc..0000000000 --- a/ush/rocoto/rocoto_viewer.py +++ /dev/null @@ -1,2383 +0,0 @@ -#!/usr/bin/env python -# -##@namespace rocoto_viewer -# @brief A Curses based terminal viewer to interact and display the status of a Rocoto Workflow in real time. -# -# @anchor rocoto_viewer -## This Python script allows users to see and interact with a running Rocoto Workflow in real time. -# \image html pythonCurses.jpeg "Rocoto Viewer for Displaying Real-time Status of Workflow" -# -# To launch this viewer simply give it the database and the XML files being used by the \b Rocoto system for your experiment: -# -# rocoto_viewer.py -w my_gfs-workflow.xml -d my_database.db -# -# The script is located in the directory para/exp/rocoto/rocotoviewers/rocotoviewer_curses/rocoto_viewer.py -# The view will continuously update every four minutes and reflect the current status of your workflow. You may use your mouse or arrow keys to select a particular task and view its status details by pressing the key \p c as indicated as \b \ (which runs \b rocotocheck) or perform a \b rocotorewind by pressing \b \ to restart the workflow at that point. Running \b rocotorewind causes the state information of that task to be cleared from the database and resubmits the job to the scheduler. -# -# Tasks marked with the \b \< symbol are \b metatasks and can be expanded by highlight that task with the mouse, and then clicking on the \b \< symbol which then changes to \b \> . You can then click on the \b \> symbol to collapse it again. Alternatively, you can select the 'x' to expand and collapse metatasks when selected. -# -##@cond ROCOTO_VIEWER_CURSES - -from __future__ import division - -import curses - -import os, sys, getpass, getopt, signal, tempfile -from os.path import basename -import subprocess -from math import * - -from __builtin__ import any as b_any -from os.path import realpath, normpath, dirname, getsize -from io import StringIO -from itertools import groupby -from time import time -from multiprocessing import Process, Queue -import time as std_time -from datetime import datetime, timedelta -import uuid -import shutil - -import sqlite3,datetime,collections -import xml.etree.ElementTree as ET -import cPickle - -try: - from dateutil.relativedelta import relativedelta -except ImportError: - #print 'dateutil which uses relativedelta to increment monthly (used by UGCS) is not supported with this version of python. Use Anaconda the native version in /user/bin' - #sys.exit(1) - pass - -# Global Variables -database_file_agmented = None -use_performance_metrics = False -default_column_length = 125 -stat_read_time_delay = 3*60 -temp_workflow_file = '' -header_string = '' -format_string = "jobid slots submit_time start_time cpu_used run_time delimiter=';'" - -ccs_html=''' - - - - - - -''' -bottom_message_scroll = 'heck oot ewind un (->) Next Cycle (<-) Previous Cycle p own elp uit' -bottom_message = 'heck oot ewind un (->) Next Cycle (<-) Previous Cycle elp uit' - -#Global Variables -#================ -list_tasks = False -html_output = False -html_output_file = None -rzdm_path = '' -only_check_point = False -save_checkfile_path = None -use_multiprocessing = True -get_user = getpass.getuser() - -screen_resized = False -debug = None - -mlines = 0 -mcols = 0 - -def sigwinch_handler(signum, frame): - global screen_resized - global mlines - global mcols - term_size = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE) - try: - get_term_size, err = term_size.communicate() - except: - return - mlines,mcols = map(int,get_term_size.split()) - screen_resized = True - -def usage(message=None): - curses.endwin() - print>>sys.stderr, ''' -Usage: rocoto_status_viewer.py -w workflow.xml -d database.db [--listtasks]\n [--html=filename.html]\n [--perfmetrics={True,False}] - -Mandatory arguments: - -w workflow.xml - -d database.db -Optional arguments: - --listtasks --- print out a list of all tasks - --html=filename.html --- creates an HTML document of status - --perfmetrics=True --- turn on/off extra columns for performance metrics - --help --- print this usage message''' - - if message is not None: - print>>sys.stderr,'\n'+str(message).rstrip()+'\n' - sys.exit(-1) - -def augment_SQLite3(filename): - - connection=sqlite3.connect(filename) - c=connection.cursor() - #qinfo=c.execute("DROP TABLE IF EXISTS jobs_augment;") - qinfo=c.execute("PRAGMA table_info(jobs_augment)").fetchall() - if any('qtime' in element for element in qinfo): - c.close() - return 'is_already_augmented' - else: - sql_create_augment_table = "CREATE TABLE jobs_augment AS SELECT * FROM jobs;" - q=c.execute(sql_create_augment_table) - q=c.execute("alter table jobs_augment add column qtime integer;") - q=c.execute("alter table jobs_augment add column cputime integer;") - q=c.execute("alter table jobs_augment add column runtime integer;") - q=c.execute("alter table jobs_augment add column slots integer;") - connection.commit() - - c.close() - database_file = filename - return 'now_augmented' - -def isSQLite3(filename): - from produtil.fileop import check_file - from produtil.fileop import deliver_file - if not check_file(filename): - return False - if getsize(filename) < 100: - return False - with open(filename, 'rb') as fd: - header = fd.read(100) - fd.close() - if not header[:16] == 'SQLite format 3\x00': - return False - else: - return True - -def isRocotoWorkflow(filename): - from produtil.fileop import check_file - if not check_file(filename): - return False - with open(filename, 'r') as input: - for line in input: - if 'DOCTYPE workflow' in line: - input.close() - return True - return False - - -def load_produtil_pythonpath(): - - try: - import produtil.cluster - return True - except ImportError: - pass - - PRODUTIL = collections.defaultdict(list) - PRODUTIL['theia'] = '/scratch4/NCEPDEV/global/save/glopara/svn/nceplibs/produtil/trunk/ush' - PRODUTIL['luna'] = '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn/nceplibs/produtil/trunk/ush' - PRODUTIL['tide'] = '/gpfs/td1/emc/global/save/emc.glopara/svn/nceplibs/produtil/trunk/ush' - PRODUTIL['gyre'] = '/gpfs/gd1/emc/global/save/emc.glopara/svn/nceplibs/produtil/trunk/ush' - try_clusters = ('theia','luna','tide','gyre') - - for cluster in try_clusters: - sys.path.append(PRODUTIL[cluster]) - try: - import produtil.cluster - return True - except ImportError: - pass - return False - -def get_arguments(): - from produtil.fileop import check_file - short_opts = "w:d:f:" - long_opts = ["checkfile=","workfolw=","database=","html=","listtasks","onlycheckpoint","help","perfmetrics="] - try: - opts, args = getopt.getopt(sys.argv[1:], short_opts, long_opts) - except getopt.GetoptError as err: - print str(err) - print - usage('SCRIPT IS ABORTING DUE TO UNRECOGNIZED ARGUMENT') - - global save_checkfile_path - global use_performance_metrics - workflow_file = None - database_file = None - perfmetrics_on = None - for k, v in opts: - if k in ('-w', '--workflow'): - workflow_file = v - elif k in ('-d','--database'): - database_file = v - elif k in ('-f','--checkfile'): - save_checkfile_path = v - elif k in ('--perfmetrics'): - perfmetrics_on = v - elif k in ('--listtasks'): - global list_tasks - list_tasks = True - elif k in ('--onlycheckpoint'): - global only_check_point - only_check_point = True - elif k in ('--html'): - global html_output - global rzdm_path - global send_html_to_rzdm - send_html_to_rzdm = True - rzdm_path = v - html_output = True - elif k in ('--help'): - usage('') - else: - pass - #usage('OPTION NOT REGOGNIZED') - - if perfmetrics_on is None: - use_performance_metrics = False - elif perfmetrics_on.lower() == 'true': - use_performance_metrics = True - elif perfmetrics_on.lower() == 'false': - use_performance_metrics = False - elif perfmetrics_on is not None: - usage('perfmetrics must be either set to true or false (e.g. --perfmetrics=True') - - send_html_to_rzdm = False - if len(rzdm_path) != 0: - if ':' not in rzdm_path or '@' not in rzdm_path: - print 'No user name or path found for sending html directory to server, no files will be sent to rzdm' - print 'Creating html folder in: %s'%rzdm_path - else: - send_html_to_rzdm = True - - if list_tasks and workflow_file is None: - usage('In order to list tasks you must supply the XML worflow-file') - - if only_check_point and (workflow_file is None or database_file is None or save_checkfile_path is None): - usage('To use the check point output you must specify the workflow, data base, and the specific name of the checkpoint file') - - if (not list_tasks) and (workflow_file is None or database_file is None): - usage('Booth database-file and workflow-file must be specified') - - if (not list_tasks) and (workflow_file is not None and database_file is not None): - #debug.write('database_file_agmented: '+database_file_agmented+'\n') - if not isSQLite3( database_file ): - usage('%s is not SQLite3 database file'%database_file) - if not isRocotoWorkflow( workflow_file ): - usage('%s is not an Rocoto XML file'%workflow_file) - - #global use_multiprocessing - #if getsize(database_file) < 104857600: - # use_multiprocessing = True - #else: - # use_multiprocessing = True - - return (workflow_file,database_file ) - - -def get_entity_values( workflow_file ): - - entity_values = collections.defaultdict(list) - with open( workflow_file, 'rw' ) as f: - for line in f: - split_line = line.split() - if ']>' in line: - break - if 'ENTITY' in line: - if 'SYSTEM' in line: - value = split_line[3] - else: - value = split_line[2] - entity_values[ split_line[1] ] = value[:-1].replace('"','') - return entity_values - -def timedelta_total_seconds(timedelta): - return ( - timedelta.microseconds + 0.0 + - (timedelta.seconds + timedelta.days * 24 * 3600) * 10 ** 6) / 10 ** 6 - -def get_aug_perf_values( username ): - from produtil.run import run,runstr, batchexe - global html_ouput - global format_keys - cmd = batchexe('which') ['bjobs'] - try: - which_bjobs = runstr(cmd).strip() - except Exception,e: - return None - bjobs = collections.defaultdict(dict) - aug_perf = collections.defaultdict(dict) - cmd = batchexe( which_bjobs )['-a','-o',format_string,'-u',username] - bjobs_line = runstr(cmd) - if 'No job found' in bjobs_line: - return None - bjobs_lines = bjobs_line.split('\n') - for l,line in enumerate(bjobs_lines): - split_line = line.split(';') - if l == 0: - format_keys = split_line - continue - for i, value in enumerate(split_line): - if i == 0: - key = value - else: - if format_keys[i] in ('RUN_TIME','CPU_USED'): - value_list = value.split() - if len(value_list) > 1: - value = value_list[0] - bjobs[key][format_keys[i]] = value - sub_time_string = '' - year = str(datetime.datetime.now().year)+' ' - sub_time = None - bstart_time = None - for jobid,keys in bjobs.iteritems(): - #debug.write(jobid+'\n') - for key in keys: - #debug.write(' '+key+":"+bjobs[jobid][key]+'\n') - try: - int_key = int(bjobs[jobid][key].strip()) - str_key = str(int_key) - except: - str_key = bjobs[jobid][key].strip() - - if key == 'SUBMIT_TIME': - sub_time_string = str_key - try: - sub_time = datetime.datetime.strptime( year+sub_time_string, '%Y %b %d %H:%M' ) - except: - sub_time = None - continue - elif key == 'START_TIME': - bstart_time_string = str_key - try: - bstart_time = datetime.datetime.strptime( year+bstart_time_string, '%Y %b %d %H:%M' ) - except: - bstart_time = None - continue - elif key == 'RUN_TIME': - aug_perf[jobid]['runtime'] = str_key - elif key == 'CPU_USED': - aug_perf[jobid]['cputime'] = str_key - elif key == 'SLOTS': - aug_perf[jobid]['slots'] = str_key - - if bstart_time_string == sub_time_string: - aug_perf[jobid]['qtime'] = '0' - elif sub_time is not None and bstart_time is None : - try: - aug_perf[jobid]['qtime'] = str(int(( datetime.datetime.now() - sub_time ).total_seconds())) - except AttributeError: - aug_perf[jobid]['qtime'] = str(int(timedelta_total_seconds( datetime.datetime.now() - sub_time ))) - - elif sub_time is not None and bstart_time is not None: - try: - aug_perf[jobid]['qtime'] = str(int((bstart_time - sub_time).total_seconds())) - except AttributeError: - aug_perf[jobid]['qtime'] = str(int(timedelta_total_seconds(bstart_time - sub_time))) - else: - aug_perf[jobid]['qtime'] = '-' - - return aug_perf - -def help_screen( screen ): - - max_row = 25 - box_cols = 60 - box = curses.newwin( max_row, box_cols , 5, 32 ) - box.box() - box.border(0) - box.addstr( 0 , 23, ' when done', curses.A_BOLD ) - helpstr= [ 'heck : run rocotocheck on selected task(s)', - 'oot : run rocotoboot on selected task(s)', - 'ewind : run rocotorewind on selected task(s)', - 'un : run rocotorun on selected task(s)', - ' ', - '(->) Next Cycle own (or) Page-dwn to scroll', - '(<-) Previous Cycle own (or) Page-up to scroll ', - ' ', - ' + Arrow Up to selected multiple tasks', - ' + Arrow Down for using with rocoto utils', - 'Double-Click or to expand/collapse metatasks', - ' ', - ' Selects a task for list or opens meta-task list', - ' ', - ' When a meta-task list is open for selection:', - ' Double-Click (or) to select the begining', - ' of a range for selection and repeate to complete', - ' the desired selected list.', - '', - 'oads and renews status data (no rocotorun)', - 'inds the last cycle with a running task', - 'nloads and clears all previously seleted tasks', - 'makes a symlink of log file of highlited task'] - - for i in range(0,len(helpstr)): - box.addstr( 1+i ,2, helpstr[i] ) - x = screen.getch() - while x != ord('q'): - x = screen.getch() - box.refresh() - -def list_selector( screen, selected_strings, strings ): - - global screen_resized - global mlines - global mcols - global highlightText - global highlightSelectedText - global normalText - - def define_box(): - - if len( strings ) < mlines: - max_row = len( strings ) - else: - max_row = mlines - 12 - max_mcols = max(18,len(max( strings, key=len ))) - if max_mcols + 8 < mcols: - box_cols = max_mcols + 8 - else: - box_cols = mcols - 3 - box = curses.newwin( max_row + 6, box_cols , 4, 5 ) - box.box() - box.border(0) - - return box, max_row, box_cols - - strings_selected = selected_strings - string_ctr_selected = '' - - box, max_row, box_cols = define_box() - row_num = len( strings ) - pages = int( ceil( row_num / max_row ) ) - position = 1 - page = 1 - for i in range( 1, max_row+1 ): - if row_num == 0: - box.addstr( 1, 1, "There aren't strings", highlightText ) - else: - print_string = ' '+strings[ i - 1 ]+' ' - if (i == position): - box.addstr( i+1, 2, print_string, highlightText ) - else: - box.addstr( i+1, 2, print_string, normalText ) - if i == row_num: - break - - screen_resized = False - - range_selected = False - string_ctr_selected_prior = '' - - x = screen.getch() - while x != ord('q'): - - if screen_resized: - - screen_resized = False - curses.resizeterm(mlines, mcols) - screen.refresh() - box.clear() - box.erase() - - box, max_row, box_cols = define_box() - - box.border( 0 ) - box.refresh() - - if x in ( curses.KEY_SF, curses.KEY_DOWN): - if x == curses.KEY_SF: - string_selected = strings[ position - 1 ] - if string_selected in strings_selected: - string_ctr_selected = '' - try: - if len(strings_selected) > 0: - strings_selected.remove( string_selected ) - except ValueError: - pass - else: - strings_selected.append( string_selected ) - if page == 1: - if position < i: - position = position + 1 - else: - if pages > 1: - page = page + 1 - position = 1 + ( max_row * ( page - 1 ) ) - elif page == pages: - if position < row_num: - position = position + 1 - else: - if position < max_row + ( max_row * ( page - 1 ) ): - position = position + 1 - else: - box.erase() - box.border(0) - page = page + 1 - position = 1 + ( max_row * ( page - 1 ) ) - if x in ( curses.KEY_SR, curses.KEY_UP): - if x == curses.KEY_SR: - string_selected = strings[ position - 1 ] - if string_selected in strings_selected: - try: - if len(strings_selected) > 0: - strings_selected.remove( string_selected ) - except ValueError: - pass - else: - strings_selected.append( string_selected ) - if page == 1: - if position > 1: - position = position - 1 - else: - if position > ( 1 + ( max_row * ( page - 1 ) ) ): - position = position - 1 - else: - box.erase() - box.border(0) - page = page - 1 - position = max_row + ( max_row * ( page - 1 ) ) - - if x == curses.KEY_PPAGE: - box.erase() - box.border( 0 ) - if page > 1: - page = page - 1 - position = 1 + ( max_row * ( page - 1 ) ) - - if x == curses.KEY_NPAGE: - box.erase() - box.border( 0 ) - #screen.refresh() - if page < pages: - page = page + 1 - position = ( 1 + ( max_row * ( page - 1 ) ) ) - - if x in ( curses.KEY_MOUSE, ord('s') ): - mouse_id, mouse_x, mouse_y, mouse_z, button_state = (0,0,0,0,0) - index_prior_selected = 0 - if x == curses.KEY_MOUSE: - mouse_id, mouse_x, mouse_y, mouse_z, button_state = curses.getmouse() - box.erase() - box.border( 0 ) - pos = mouse_y-5 - if page == 1: - position = pos - else: - position = max_row*(page-1)+pos - - if x == ord('s') or (button_state & curses.BUTTON1_DOUBLE_CLICKED): - string_ctr_selected = strings[ position - 1 ] - if range_selected: - range_selected = False - string_ctr_selected = '' - if string_ctr_selected != string_ctr_selected_prior: - index_prior_selected = strings.index(string_ctr_selected_prior) - if position < index_prior_selected: - first = position-1 - last = index_prior_selected+1 - else: - first = index_prior_selected - last = position - for i in range( first, last ): - if strings[i] in strings_selected: - strings_selected.remove(strings[i]) - else: - strings_selected.append( strings[i] ) - string_ctr_selected_prior = '' - else: - range_selected = True - string_ctr_selected_prior = string_ctr_selected - - if x in (curses.KEY_ENTER, 10, 13) and row_num != 0: - box.border( 0 ) - string_selected = strings[ position - 1 ] - if string_ctr_selected_prior == string_selected: - string_ctr_selected_prior = '' - range_selected = False - if string_selected in strings_selected: - try: - if len(strings_selected) > 0: - strings_selected.remove( string_selected ) - except ValueError: - pass - else: - strings_selected.append( string_selected ) - - if x == ord('U'): - for each_sting in strings: - if each_sting in strings_selected: - if len(strings_selected) > 0: - strings_selected.remove(each_sting) - - for i in range( 1 + ( max_row * ( page - 1 ) ), max_row + 1 + ( max_row * ( page - 1 ) ) ): - if row_num == 0: - box.addstr( 1, 1, "There aren't strings", highlightText ) - else: - if strings[ i - 1 ] == string_ctr_selected_prior: - string_print = '* '+strings[ i - 1 ]+' ' - else: - string_print = ' '+strings[ i - 1 ]+' ' - - start_pos = i - ( max_row * ( page - 1 ) ) + 1 - if ( i + ( max_row * ( page - 1 ) ) == position + ( max_row * ( page - 1 ) ) ): - box.addstr( start_pos, 2, string_print, highlightText ) - else: - box.addstr( start_pos, 2, string_print, normalText ) - if strings[ i - 1 ] in strings_selected: - box.addstr( start_pos, 2, string_print[:1] ) - box.addstr( start_pos, 4, string_print[2:-1], highlightSelectedText | curses.A_DIM ) - if i == row_num: - break - - box.addstr( max_row+3 , 2, 'Select with or' ) - box.addstr( max_row+4 , 2, ' + ' ) - box.addstr( 0 , 7, ' when done', curses.A_BOLD ) - box.refresh() - x = screen.getch() - - return strings_selected - - -def get_rocoto_check(params, queue_check): - from produtil.run import run,runstr, batchexe, exe - workflow_file, database_file, task, cycle, process = params - cmd=batchexe('rocotocheck')['-v',10,'-w',workflow_file,'-d',database_file,'-c',cycle,'-t',task] - check=runstr(cmd) - if check is None: - curses.endwin() - print 'rcotocheck falied: %d'%stat - sys.exit(-1) - queue_check.put(check) - -def rocoto_boot(params): - from produtil.run import run,runstr, batchexe, exe - workflow_file, database_file, cycle, metatask_list, task_list = params - run( exe('yes') | exe('head')['-1'] > '.yes.txt') - if len(task_list) == 0 and len(metatask_list) != 0: - cmd=batchexe('rocotoboot')['--workflow', workflow_file,'--database',database_file,'--cycles',cycle,'--metatasks', metatask_list] < '.yes.txt' - elif len(task_list) != 0 and len(metatask_list) == 0: - cmd=batchexe('rocotoboot')['--workflow', workflow_file,'--database',database_file,'--cycles',cycle,'--tasks', task_list ] < '.yes.txt' - elif len(task_list) != 0 and len(metatask_list) != 0: - cmd=batchexe('rocotoboot')['--workflow', workflow_file,'--database',database_file,'--cycles',cycle,'--tasks', task_list, '--metatasks', metatask_list ] < '.yes.txt' - else: - return 'Warning: No metatasks or tasks where selected when rocotboot was called' - stat=runstr(cmd) - if stat is None: - display_results( 'rcotoboot falied!!','') - return stat - -def rocoto_rewind(params): - from produtil.run import run,runstr, batchexe - workflow_file, database_file, cycle, process = params - cmd=batchexe('rocotorewind')['-w',workflow_file,'-d',database_file,'-c',cycle,process] - stat=runstr(cmd) - if stat is None: - display_results('rcotorewind falied!!','') - return stat - -def rocoto_run(params): - from produtil.run import run,runstr, batchexe - workflow_file, database_file = params - cmd=batchexe('rocotorun')['-w',workflow_file,'-d',database_file] - stat=runstr(cmd ) - stat = '' - if stat is None: - curses.endwin() - print 'rcotorun falied: %d'%stat - sys.exit(-1) - return stat - -def get_tasklist(workflow_file): - import produtil.run, produtil.numerics - tasks_ordered = [] - metatask_list = collections.defaultdict(list) - tree = ET.parse(workflow_file) - root = tree.getroot() - cycledef_group_cycles = collections.defaultdict(list) - if list_tasks: - curses.endwin() - print - cycle_noname = 'default_cycle' - for child in root: - if child.tag == 'cycledef': - if len(child.attrib) != 0: - cycle_def_name = child.attrib['group'] - else: - cycle_def_name = cycle_noname - cycle_string = child.text.split() - - ucgs_is_cron = None - if PACKAGE.lower() == 'ugcs': - start_cycle = produtil.numerics.to_datetime ( entity_values['SDATE'] ) - end_cycle = produtil.numerics.to_datetime ( entity_values['EDATE'] ) - #inc_cycle = produtil.numerics.to_timedelta( entity_values['INC_MONTHS'] ) - #NOTE: this is for the special case when cycle for every month - inc_cycle = int(entity_values['INC_MONTHS']) - if inc_cycle == 0: - inc_cycle = produtil.numerics.to_timedelta( cycle_string[2] ) - ucgs_is_cron = False - else: - ucgs_is_cron = True - only_once_ugcs = True - else: - start_cycle = produtil.numerics.to_datetime ( cycle_string[0] ) - end_cycle = produtil.numerics.to_datetime ( cycle_string[1] ) - inc_cycle = produtil.numerics.to_timedelta( cycle_string[2] ) - - while start_cycle <= end_cycle: - cycledef_group_cycles[cycle_def_name].append(start_cycle.strftime("%Y%m%d%H%M")) - if PACKAGE.lower() == 'ugcs' and ucgs_is_cron: - try: - start_cycle = start_cycle + relativedelta(months=+inc_cycle) - except AttributeError: - curses.endwin() - print;print - print 'dateutil which uses relativedelta to increment monthly (used by UGCS) is not supported with this version of python.\nUse Anaconda the native version in /user/bin' - sys.exit(-1) - else: - start_cycle = start_cycle + inc_cycle - #if list_tasks: - #print 'cycledef=%s number of cycles %s inc: %s'%(cycle_def_name, len(cycledef_group_cycles[cycle_def_name]),inc_cycle) - #print 'contails cycles',cycledef_group_cycles[cycle_def_name] - if child.tag == 'task': - task_name = child.attrib['name'] - log_file = child.find('join').find('cyclestr').text.replace( '@Y@m@d@H','CYCLE' ) - #if len(log_file) != 0: - # print 'LOG: %s %s'%( task_name, log_file ) - if 'cycledefs' in child.attrib: - task_cycledefs = child.attrib['cycledefs'] - #if list_tasks: - # print 'task_cycledefs:',task_cycledefs - else: - task_cycledefs = cycle_noname - if list_tasks: - print task_name,task_cycledefs - #dependancies = child.getiterator('dependency') - #for dependency in dependancies: - # for them in dependency.getchildren(): - # print them.attrib - tasks_ordered.append((task_name,task_cycledefs,log_file)) - elif child.tag == 'metatask': - all_metatasks_iterator = child.getiterator('metatask') - all_vars = dict() ; all_tasks = [] - for i,metatasks in enumerate(all_metatasks_iterator): - metatask_name = 'NO_NAME' - try: - metatask_name = metatasks.attrib['name'] - except: - pass - if list_tasks: - print ' '*i+'metatask:',metatask_name - all_vars_list = metatasks.findall('var') - all_tasks_list = metatasks.findall('task') - for var in all_vars_list: - var_list_values = var.text.split() - #print ' '+' '*i+'(%d) var name:'%i,var.attrib['name'],var_list_values - all_vars[var.attrib['name']] = var_list_values - for task in all_tasks_list: - task_name = task.attrib['name'] - task_log = task.find('join').find('cyclestr').text.replace( '@Y@m@d@H','CYCLE' ) - #if len(task_log) != 0: - # print 'LOG: %s %s'%( task_name, task_log) - #print ' '+' '*i+'(%d) task name:'%i,task.attrib['name'] - if 'cycledefs' in task.attrib: - task_cycledefs = task.attrib['cycledefs'] - #if list_tasks: - # print 'task_cycledefs (meta):',task_cycledefs - else: - task_cycledefs = cycle_noname - all_tasks.append((task_name,task_cycledefs,task_log)) - add_task = [] - for task_name in all_tasks: - first_task_resolved = False - first_task_resolved_name = '' - add_task[:] = [] - add_task.append(task_name) - for name,vars in all_vars.iteritems(): - replace_var = '#'+name+'#' - #print 'TASK_NAME: %s | %s'%(task_name,replace_var) - for each_task_name in add_task: - #for each_task_name in all_tasks: - if replace_var in each_task_name[0]: - for var in vars: - new_task_name = each_task_name[0].replace(replace_var, var) - new_task_log = each_task_name[2].replace(replace_var, var) - add_task.append((new_task_name,each_task_name[1],new_task_log)) - for task in add_task: - if '#' not in task[0]: - if task[0] not in [ j[0] for j in tasks_ordered]: - tasks_ordered.append(task) - if not first_task_resolved: - first_task_resolved = True - first_task_resolved_name = task[0] - if metatask_name == 'NO_NAME': - metatask_list[task[0]].append(task[0]) - else: - metatask_list[task[0]].append(metatask_name) - metatask_list[task[0]].append(task[0]) - else: - metatask_list[first_task_resolved_name].append(task[0]) - if list_tasks: - print ' '+' '*i+task[0],task[1],'LOG:',task[2] - - # Default expantion of metatasks True = collapsed - #for metatask,metatasks in metatask_list.iteritems(): - # metatask_list[metatask].append(True) - - return tasks_ordered,metatask_list,cycledef_group_cycles - -def get_rocoto_stat(params, queue_stat): - workflow_file, database_file, tasks_ordered, metatask_list, cycledef_group_cycles = params - - global temp_workflow_file - global database_file_agmented - if len(tasks_ordered) == 0 or len(metatask_list) == 0 or len(cycledef_group_cycles) == 0 or list_tasks: - tasks_ordered, metatask_list,cycledef_group_cycles = get_tasklist(temp_workflow_file) - - if use_performance_metrics: - aug_perf = get_aug_perf_values(get_user) - else: - aug_perf = None - - info=collections.defaultdict(list) - cycles=set() - - connection=sqlite3.connect(database_file) - c=connection.cursor() - - if use_performance_metrics: - q=c.execute("DROP TABLE IF EXISTS jobs_augment_tmp;") - sql_create_augment_table = "CREATE TABLE jobs_augment_tmp AS SELECT * FROM jobs;" - q=c.execute(sql_create_augment_table) - q=c.execute("alter table jobs_augment_tmp add column qtime integer;") - q=c.execute("alter table jobs_augment_tmp add column cputime integer;") - q=c.execute("alter table jobs_augment_tmp add column runtime integer;") - q=c.execute("alter table jobs_augment_tmp add column slots integer;") - - sq_command = '' - column_updates = ('qtime','cputime','runtime','slots') - sqlite_merge_command = "%s=(SELECT jobs_augment.%s FROM jobs_augment WHERE jobs_augment.id=jobs_augment_tmp.id)" - for column in column_updates: - sq_command += sqlite_merge_command%(column,column)+',' - sq_command=';'.join(sq_command.rsplit(',', 1)) - sq_command = 'UPDATE jobs_augment_tmp SET '+sq_command - q=c.execute(sq_command) - - sq_command = 'UPDATE jobs_augment_tmp SET ' - sqlite_update_command = "%s = '%s' WHERE jobs_augment_tmp.jobid = %s" - #debug.write('WRITING TO DATABASE'+'\n') - for perf_jobid,perf_values in aug_perf.iteritems(): - for name,each_value in perf_values.iteritems(): - q=c.execute(sq_command+sqlite_update_command%(name,each_value,perf_jobid)) - #debug.write('SQL: '+sq_command+sqlite_update_command%(name,each_value,perf_jobid+'\n')) - - qinfo=c.execute("DROP TABLE IF EXISTS jobs_augment;") - qinfo=c.execute("ALTER TABLE jobs_augment_tmp RENAME TO jobs_augment;") - - cycledifitions = [] - q=c.execute('SELECT id, groupname, cycledef FROM cycledef') - for row in q: - (theid, groupname, cycledef) = row - cycledifitions.append( (theid, groupname, cycledef) ) - - cycle_done_stat = dict() - q=c.execute('SELECT id,cycle,done FROM cycles') - for row in q: - (theid,cycle,done)=row - cycles.add(cycle) - cycle_done_stat[cycle]=done - - if use_performance_metrics: - q=c.execute('SELECT id,jobid,taskname,cycle,state,exit_status,duration,tries,qtime,cputime,runtime,slots FROM jobs_augment') - else: - q=c.execute('SELECT id,jobid,taskname,cycle,state,exit_status,duration,tries FROM jobs') - - q_get = [] - entered_jobids = [] - last_task_index = 0 - for row in q: - row = tuple('-' if x is None else x for x in row) - if use_performance_metrics: - (theid, jobid,taskname,cycle,state,exit_status,duration,tries,qtime,cputime,runtime,slots)=row - else: - (theid, jobid,taskname,cycle,state,exit_status,duration,tries,)=row - if jobid in entered_jobids: - continue - else: - if taskname in tasks_ordered: - task_index = [x[0] for x in task_ordered].index(taskname) - #task_index = tasks_ordered.index(taskname) - last_task_index = task_index - else: - task_index = last_task_index - - if use_performance_metrics: - q_get.append( (theid,jobid,task_index,taskname,cycle,state,exit_status,duration,tries,qtime,cputime,runtime,slots) ) - else: - q_get.append( (theid,jobid,task_index,taskname,cycle,state,exit_status,duration,tries) ) - entered_jobids.append(jobid) - - q_get.sort( key=lambda x: x[2] ) - - connection.commit() - c.close() - - for row in q_get: - if use_performance_metrics: - (theid,jobid,task_order,taskname,cycle,state,exit_status,duration,tries,qtime,cputime,runtime,slots)=row - else: - (theid,jobid,task_order,taskname,cycle,state,exit_status,duration,tries)=row - if jobid != '-': - if use_performance_metrics: - line = '%s %s %s %s %s %s %s %s %s %s %s'%(datetime.datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M'),taskname,str(jobid),str(state),str(exit_status),str(tries),str(duration).split('.')[0],str(slots),str(qtime),str(cputime).split('.')[0],str(runtime)) - else: - line = '%s %s %s %s %s %s %s'%(datetime.datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M'),taskname,str(jobid),str(state),str(exit_status),str(tries),str(duration).split('.')[0]) - #debug.write('LINE: '+line+'\n') - info[cycle].append(line) - - for every_cycle in cycles: - if len(info[every_cycle]) == 0: - info[every_cycle].append('place holder') - - new_info=collections.defaultdict(list) - job_ids = [] - job_id = '' - for each_cycle,lines_in_cycle in info.iteritems(): - for task in tasks_ordered: - skip_task = False - for each_line in lines_in_cycle: - if task[0] == each_line.split()[1]: - #if task[0]+' ' in each_line: - job_id = each_line.split()[2] - if job_id in job_ids: - break - cycle_string = datetime.datetime.fromtimestamp(each_cycle).strftime('%Y%m%d%H%M') - #print 'TESTB:', len(task), task[0],task[1] - cycledefs = task[1].split(',') - if len(cycledefs) > 1: - #print 'Checking if %s for %s is in a gfs cycle:'%(task[0],cycle_string) - for each_cycledef in cycledefs: - #print 'group:', each_cycledef, cycledef_group_cycles[each_cycledef] - if cycle_string in cycledef_group_cycles[each_cycledef]: - #print 'Found:', task[0],'with cycle',cycle_string - new_info[each_cycle].append(each_line) - job_ids.append(job_id) - skip_task = True - break - elif cycle_string in cycledef_group_cycles[task[1]]: - new_info[each_cycle].append(each_line) - job_ids.append(job_id) - skip_task = True - break - if skip_task: - continue - line = datetime.datetime.fromtimestamp(each_cycle).strftime('%Y%m%d%H%M')+' '*7+task[0]+' - - - - -' - cycle_string = datetime.datetime.fromtimestamp(each_cycle).strftime('%Y%m%d%H%M') - cycledefs = task[1].split(',') - if len(cycledefs) > 1: - for each_cycledef in cycledefs: - if cycle_string in cycledef_group_cycles[each_cycledef]: - new_info[each_cycle].append(line) - skip_task = True - break - elif cycle_string in cycledef_group_cycles[task[1]]: - new_info[each_cycle].append(line) - skip_task = True - if skip_task: - continue - - rocoto_stat = [] - for cycle in sorted(cycles): - if len(new_info[cycle]) != 0: - rocoto_stat.append(new_info[cycle]) - - if save_checkfile_path is not None: - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - with open(save_checkfile_path, 'w') as savefile: - rocoto_data_and_time = (rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles, stat_update_time) - cPickle.dump(rocoto_data_and_time, savefile) - if only_check_point: - sys.exit(0) - - if use_multiprocessing: - queue_stat.put((rocoto_stat, tasks_ordered, metatask_list, cycledef_group_cycles)) - else: - return (rocoto_stat, tasks_ordered, metatask_list, cycledef_group_cycles) - - -def display_results(results,screen,params): - from produtil.fileop import check_file - results_lines = results.split('\n') - num_lines,num_columns = (len(results_lines)+3,len(max(results_lines, key=len))+1) - pad_pos = 0 - force_load_stat = False - global mlines - global mcols - while True: - screen.clear() - screen.refresh() - results_pad = curses.newpad(num_lines,num_columns) - for results_line in results_lines: - results_pad.addstr(results_line+'\n') - results_pad.refresh( pad_pos, 0, 0,0, mlines-3,mcols-1) - extra_1 = extra_2 = '' - if pad_pos < num_lines-mlines-2 or pad_pos > 0: - extra_1 = '/ Scroll' - if len(params) != 0: - extra_2 = 'ave results to a file' - screen.addstr(mlines-1,0,' Return %s %s'%(extra_1,extra_2),curses.A_BOLD) - event = screen.getch() - if event == curses.KEY_RESIZE: - screen.refresh() - elif event in ( curses.KEY_PPAGE, ord('u') ): - if pad_pos < num_lines-mlines-2: - pad_pos += 1 - elif event in ( curses.KEY_NPAGE, ord('d') ): - if pad_pos != 0: - pad_pos -= 1 - elif event == curses.KEY_ENTER or event == 10: - screen.clear() - break - elif event == ord('s'): - strg = [] - strg.append(PSLOT) - for i in range(2,5): - try: - if ' ' not in basename(params[i]): - strg.append(basename(params[i]).split('.')[0]) - except: - pass - if len(strg) == 0: - strg = 'rocotoviewer_outout_file' - save_results_file = '_'.join(strg)+'.txt' - inc_int = 0 - while check_file(save_results_file): - if '(%d)'%inc_int in save_results_file: - save_results_file = save_results_file.replace('(%d)'%inc_int,'(%d)'%(inc_int+1)) - inc_int += 1 - else: - save_results_file = basename(save_results_file.split('.')[0])+'(%d)'%inc_int+'.txt' - out_file = open(save_results_file,'w') - out_file.write(results) - out_file.close() - screen.addstr(mlines-1,0,'Saved file %s'%save_results_file+' '*10) - screen.refresh() - std_time.sleep(0.5) - - return - -def main(screen): - - global mlines - global mcols - global default_column_length - global use_multiprocessing - global highlightText - global highlightSelectedText - global normalText - global PSLOT - global PACKAGE - global entity_values - - event = 10 - - if not sys.stdin.isatty(): - if screen != 'dummy': - print 'There seems to be a problem with the curses init' - sys.exit(-1) - else: - mlines = 100 - else: - mlines, mcols = screen.getmaxyx() - - #global debug - #PWD = os.getcwd() - #debug = open(PWD+'/debug.log','a',0) - - (workflow_file,database_file) = get_arguments() - - if not load_produtil_pythonpath(): - curses.endwin() - print '\n\nCRITICAL ERROR: The produtil package could not be loaded from your system' - sys.exit(-1) - - if html_output: - if sys.stdin.isatty(): - curses.endwin() - print '\nPreparing to write out an html folder' - use_multiprocessing = False - - import produtil.run, produtil.numerics - from produtil.run import run,runstr, batchexe - from produtil.fileop import check_file, makedirs, deliver_file, remove_file, make_symlinks_in - from produtil.prog import shbackslash - - header_string = ' CYCLE TASK JOBID STATE EXIT TRIES DURATION' - header_string_under = '========(updated:tttttttttttttttt)========== PSLOT: pslot ===========================' - - global use_performance_metrics - aug_perf = collections.defaultdict(dict) - if use_performance_metrics: - result = augment_SQLite3( database_file ) - aug_perf = get_aug_perf_values(get_user) - header_string += ' SLOTS QTIME CPU RUN\n' - header_string_under += '=============================\n' - header_string += header_string_under - default_column_length = 122 - else: - aug_perf = None - header_string = header_string+'\n'+header_string_under+'\n' - default_column_length = 91 - - html_output_dir = None - entity_values = get_entity_values( workflow_file ) - workflow_name = 'gfs_workflow' - if 'ROTDIR' in entity_values: - ROTDIR = entity_values['ROTDIR'] - else: - ROTDIR = 'no_rotdir' - if 'PSLOT' in entity_values: - PSLOT = entity_values['PSLOT'] - else: - PSLOT = 'no_name' - if 'PACKAGE' in entity_values: - PACKAGE = entity_values['PACKAGE'] - if PACKAGE == 'ugcs': - workflow_name = 'ugcs_workflow' - if PACKAGE == 'gfs': - workflow_name = 'gfs_workflow' - else: - PACKAGE = 'none' - if 'EXPDIR' in entity_values: - EXPDIR = entity_values['EXPDIR'] - else: - EXPDIR = '.' - - if html_output: - html_ptr = None - if not send_html_to_rzdm and len(rzdm_path) != 0: - html_output_dir = shbackslash(rzdm_path) - else: - html_output_dir = shbackslash('%s/pr%s'%(workflow_name,PSLOT)) - print 'writing html to directory:',html_output_dir - html_output_file = shbackslash( html_output_dir+'/index.html' ) - html_header_line = '\n' - if use_performance_metrics: - html_header_line = html_header_line+''+'\n' - else: - html_header_line = html_header_line+'\n' - print 'Generating html folder html: %s ...'%html_output_file - cmd = batchexe('rm') ['-Rf', html_output_dir ] - stat=runstr(cmd) - makedirs( html_output_dir ) - html_ptr = open(html_output_file,'w') - html_ptr.write(ccs_html) - break_file = False - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - html_discribe_line = '\n
CYCLETASKJOBIDSTATEEXITTRIESDURATIONSLOTSQTIMECPURUN
\n\n\n'%(stat_update_time,PSLOT) - html_discribe_line += '\n\n
ExpandRefreshed: %sPSLOT: %s
ROTDIR: %sTurn Around Times
\n
\n'%(workflow_name,ROTDIR,PSLOT) - html_discribe_line += html_header_line - html_ptr.write( html_discribe_line ) - else: - curses.start_color() - curses.use_default_colors() - screen.refresh() - curses.mousemask(1) - curses.noecho() - for i in range(0, curses.COLORS): - curses.init_pair(i + 1, i,curses.COLOR_BLACK) - if i == 4: - curses.init_pair(i + 1, i,curses.COLOR_WHITE) - curses.init_pair(8, 0, -1) - - curses.mousemask(curses.ALL_MOUSE_EVENTS) - #curses.init_pair(6,curses.COLOR_BLACK, curses.COLOR_CYAN) - highlightText = curses.A_STANDOUT - highlightSelectedText = curses.color_pair(5) - normalText = curses.A_NORMAL - - cmd = batchexe('which') ['rocotorun'] - try: - which_rocoto = runstr(cmd).strip() - except Exception,e: - curses.endwin() - print '\n\nCRITICAL ERROR: rocotorun is not in your path, user "module load rocoto"' - sys.exit(0) - - os.environ['TZ']='UTC' - std_time.tzset() - - #stdout_buff = StringIO() - #stderr_buff = StringIO() - #sys.stdout = stdout_buff - #sys.stderr = stderr_buff - - HOME = os.environ['HOME'] - rocoto_temp = HOME+'/.rocoto/tmp' - makedirs( rocoto_temp ) - - global temp_workflow_file - workflow_basename = basename(workflow_file)+'.' - temp_file= tempfile.NamedTemporaryFile(prefix=workflow_basename, dir=rocoto_temp, delete=False) - temp_workflow_file = temp_file.name - old = open(workflow_file) - temp = [] - for line in old: - if '&ENV_VARS;' not in line: - temp.append(line) - - for line in temp: - temp_file.write(line) - - temp_file.close() - old.close() - - tasks_ordered = [] - metatask_list = collections.defaultdict(list) - cycledef_group_cycles = collections.defaultdict(list) - - queue_stat = Queue() - queue_check = Queue() - - if only_check_point: - curses.endwin() - sys.stdout = os.fdopen(0,'w',0) - print 'Creating check point file ...' - params = (workflow_file, database_file, tasks_ordered, metatask_list, cycledef_group_cycles ) - get_rocoto_stat( params, queue_stat ) - - stat_update_time = '' - params_check = '' - header = None - - process_get_rocoto_stat = None - process_get_rocoto_check = None - - cycle = 0 - if html_output: - mlines = 100 - mcols = 125 - if not html_output and mcols < default_column_length: - curses.endwin() - print - print 'Your terminal is only %d characters must be at least %d to display workflow status'%(mcols,default_column_length) - sys.exit(-1) - if not html_output: - screen.refresh() - rocoto_stat_params = '' - rocoto_stat_params_tmp = '' - step = 0.0 ; i = 0 - dots = ('. ','.. ','... ','.... ','.....',' ....',' ...',' .') - dot_stat = 0 ; dot_check = 0 - current_time = time() - meta_tasklist = collections.defaultdict(list) - - if save_checkfile_path is not None and check_file(save_checkfile_path): - with open(save_checkfile_path) as savefile: - rocoto_data_and_time = cPickle.load(savefile) - rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles, stat_update_time = rocoto_data_and_time - start_time = time() - stat_read_time_delay - 10 - header = header_string - header = header.replace('t'*16,stat_update_time) - if PACKAGE.lower() == 'ugcs': - header = header.replace(' PSLOT: pslot ','==== UGCS ====') - elif PSLOT.lower() == 'no_name': - header = header.replace(' PSLOT: pslot ','==============') - reduce_header_size = 0 - else: - header = header.replace(' PSLOT: pslot ','==== UGCS ====') - reduce_header_size = 0 - if reduce_header_size > 0: - header = header[:-reduce_header_size] - header = header[reduce_header_size:] - if list_tasks: - params = (workflow_file, database_file, tasks_ordered, metatask_list, cycledef_group_cycles ) - get_rocoto_stat( params, Queue() ) - curses.endwin() - sys.stdout = os.fdopen(0,'w',0) - sys.exit(0) - - - if save_checkfile_path is None or (save_checkfile_path is not None and not check_file(save_checkfile_path)): - params = (workflow_file, database_file, tasks_ordered, metatask_list,cycledef_group_cycles) - if use_multiprocessing: - process_get_rocoto_stat = Process( target=get_rocoto_stat, args=[params, queue_stat] ) - process_get_rocoto_stat.start() - screen.addstr(mlines-2,0,'No checkpoint file, must get rocoto stats please wait',curses.A_BOLD) - screen.addstr(mlines-1,0,'Running rocotostat ',curses.A_BOLD) - else: - (rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles) = get_rocoto_stat( params, Queue() ) - header = header_string - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - header = header.replace('t'*16,stat_update_time) - if PSLOT.lower() == 'no_name': - header = header.replace(' PSLOT: pslot ','==============') - reduce_header_size = 0 - elif PACKAGE.lower() == 'ugcs': - header = header.replace(' PSLOT: pslot ','==== UGCS ====') - reduce_header_size = 0 - else: - header = header.replace('pslot',PSLOT) - reduce_header_size = int((len(PSLOT)-len('PSLOT'))/2) - if reduce_header_size > 0: - header = header[:-reduce_header_size] - header = header[reduce_header_size:] - - while use_multiprocessing: - if mcols < default_column_length: - curses.endwin() - print - print 'Your terminal is only %d characters must be at least %d to display workflow status'%(mcols,default_column_length) - sys.exit(-1) - step += 0.001 - if step > 100: - step = 0.0 - i = (0 if i == len(dots)-1 else i+1 ) - curses.curs_set(0) - screen.addstr(mlines-1,19,dots[i],curses.A_BOLD) - screen.refresh() - try: - rocoto_stat_params = queue_stat.get_nowait() - except: - pass - if len(rocoto_stat_params) != 0: - (rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles) = rocoto_stat_params - if use_multiprocessing: - process_get_rocoto_stat.join() - process_get_rocoto_stat.terminate() - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - header = header_string - header = header.replace('t'*16,stat_update_time) - if PSLOT.lower() == 'no_name': - header = header.replace(' PSLOT: pslot ','==============') - reduce_header_size = 0 - elif PACKAGE.lower() == 'ugcs': - header = header.replace(' PSLOT: pslot ','==== UGCS ====') - reduce_header_size = 0 - else: - header = header.replace('pslot',PSLOT) - reduce_header_size = int((len(PSLOT)-len('PSLOT'))/2) - if reduce_header_size > 0: - header = header[:-reduce_header_size] - header = header[reduce_header_size:] - break - - start_time = time() - - num_cycle = len(rocoto_stat) - time_to_load = (time()- current_time)/60.0 - - pad_pos = 0 - update_pad = True - task = 0 ; execute_task = '' ; execute_cycle = '' - loading_stat = False - loading_check = False - find_next = 0 - check_task = '' ; check_cycle = '' - rocoto_check = '' - break_twice = False - search_string = '' - - meta_tasks = [] - metatasks_state_cycle = [] - metatasks_state_string_cycle = [] - - metatask_list_copy = collections.defaultdict(list) - metatask_name = collections.defaultdict(list) - for each_metatask in metatask_list: - metatask_name[each_metatask] = metatask_list[each_metatask][0] - del metatask_list[each_metatask][0] - - tasks_in_cycle = [] - for each_cycle in rocoto_stat: - list_of_tasks_per_cycle = [] - meta_tasks_in_cycle = [] - for each_line in each_cycle: - line_has_metatask = False - for check_metatask, check_metatask_list in metatask_list.iteritems(): - if check_metatask in each_line: - meta_tasks_in_cycle.append( (check_metatask, True, check_metatask_list ) ) - line_has_metatask = True - continue - else: - for every_meta_task in check_metatask_list: - each_element_in_line = each_line.split() - if every_meta_task != check_metatask: - for item in each_element_in_line: - if every_meta_task == item: - meta_tasks_in_cycle.append((every_meta_task, False, check_metatask) ) - line_has_metatask = True - if not line_has_metatask: - if '---' not in each_line.split()[1]: - list_of_tasks_per_cycle.append(each_line.split()[1]) - meta_tasks_in_cycle.append(('False',False,'False')) - - tasks_in_cycle.append(list_of_tasks_per_cycle) - - meta_tasks_state = dict() - meta_tasks_state_string = dict() - for check_metatask, check_metatask_list in metatask_list.iteritems(): - meta_tasks_state[check_metatask] = True - meta_tasks_state_string[check_metatask] = '' - meta_tasks_state['False'] = False - - meta_tasks.append(meta_tasks_in_cycle) - metatasks_state_cycle.append(meta_tasks_state) - metatasks_state_string_cycle.append(meta_tasks_state_string) - - update_metatask_state_status_message = True - ''' -# This lists each metatask and its elements -# for the first cycle for code edification - curses.endwin() - print - for each_metatask in meta_tasks[0]: - if each_metatask[1]: - print metatask_name[each_metatask[2][0]] - for task in each_metatask[2]: - print '',task - sys.exit(0) - ''' - - metatask_list_per_cycle = [] - metatask_list_by_name = collections.defaultdict(dict) - for each_cycle in meta_tasks: - list_of_metatasks_in_cycle = [] - for each_metatask in each_cycle: - if each_metatask[1]: - tasks_in_metatask_list = [] - for task in each_metatask[2]: - tasks_in_metatask_list.append( task ) - metatask_list_by_name[ metatask_name[each_metatask[2][0]] ] = tasks_in_metatask_list - list_of_metatasks_in_cycle.append( metatask_name[each_metatask[2][0]] ) - metatask_list_per_cycle.append(list_of_metatasks_in_cycle) - - found = False - end_found = False - found_cycle = 0 - found_end_cycle = 0 - for find_cycle in range(0,len(rocoto_stat)): - for lines in rocoto_stat[find_cycle]: - if not found and any(x in lines for x in ['RUNNING', 'QUEUED']): - found = True - found_cycle = find_cycle - if found and not any(x in lines for x in ['RUNNING', 'QUEUED']): - end_found = True - found_end_cycle = find_cycle - break - - get_number_of_stats = 0 - if found: - cycle = found_cycle - else: - get_number_of_stats = 2 - if len(rocoto_stat) > 2: - cycle = len(rocoto_stat) - 2 - else: cycle = 0 - - if html_output: - if cycle > 2: - cycle -= 2 - html_start_cycle = cycle - - html_output_firstpass = True - #debug.write('num cycles: %s\n'%str(len(rocoto_stat))) - while True: - num_columns = default_column_length - mlines = 90; mcols = 125 - if header is None: - header = ' ' - if update_pad is True: - #debug.write('cycle: %s\n'%str(cycle)) - num_lines = len(rocoto_stat[cycle]) - #debug.write('len rocoto_stat[cycle]: %s\n'%str(num_lines)) - line_correction = 0 - for count_meta_tasks in meta_tasks[cycle]: - if count_meta_tasks[1] and metatasks_state_cycle[cycle][ count_meta_tasks[0] ]: - line_correction += len(count_meta_tasks[2]) - 1 - num_lines -= line_correction - update_pad = False - line_number = -1 - colapsed_metatask = False - for line_num,line in enumerate(rocoto_stat[cycle]): - columns = line.split() - count_columns = line.split(' ') - spaces = [] - for c,sub_group in groupby(count_columns): - if c != '': continue - spaces.append(' '*len(list(sub_group))) - spaces.append('') - text_color = {'SUCCEEDED':3,'QUEUED':4,'DEAD':2,'FAILED':2,'RUNNING':6} - skip_task = False - - if not meta_tasks[cycle][line_num][1] and metatasks_state_cycle[cycle][ meta_tasks[cycle][line_num][2] ] : - skip_task = True - else: - line_number +=1 - html_line = '' - if use_performance_metrics and len(columns) == 7: - for i in range(0,4): - columns.append('-') - for i,column in enumerate(columns): - if skip_task: continue - if not use_performance_metrics and i > 7: continue - execute_cycle = columns[0] - if i == 0: - if meta_tasks[cycle][line_num][1]: - if metatasks_state_cycle[cycle][columns[1]]: - colapsed_metatask = True - if update_metatask_state_status_message or len(metatasks_state_string_cycle[cycle][ columns[1] ])==0: - get_state_list = [] - total_numer_of_tasks = len(meta_tasks[cycle][line_num][2]) - for check_metatask_line in rocoto_stat[cycle]: - split_check_metatask_line = check_metatask_line.split() - for each_metatask in meta_tasks[cycle][line_num][2]: - if each_metatask == split_check_metatask_line[1]: - get_state_list.append(split_check_metatask_line[3]) - metatask_state = columns[3] - if 'SUCCEEDED' in get_state_list: - metatask_state = '(%d/%d) SUCCEEDED'%(get_state_list.count('SUCCEEDED'),total_numer_of_tasks) - if 'QUEUED' in get_state_list: - metatask_state = '(%d/%d) QUEUED'%(get_state_list.count('QUEUED'),total_numer_of_tasks) - if 'RUNNING' in get_state_list: - metatask_state = '(%d/%d) RUNNING'%(get_state_list.count('RUNNING'),total_numer_of_tasks) - if 'DEAD' in get_state_list: - metatask_state = '(%d/%d) DEAD'%(get_state_list.count('DEAD'),total_numer_of_tasks) - metatasks_state_string_cycle[cycle][ columns[1] ] = metatask_state - html_line += ''+column+'' - elif i == 1: - save_column = column - if colapsed_metatask: - colapsed_metatask = False - column = metatask_name[column] - display_column = (column if len(column) < 40 else column[:40]) - if line_number == task: - execute_task = save_column - if html_output: - log_file = '' - for find_task in tasks_ordered: - if find_task[0] == column: - log_file = find_task[2].replace('CYCLE', execute_cycle[:-2] ) - if check_file(shbackslash( log_file )): - deliver_file( log_file, html_output_dir ) - log_file_base = os.path.basename(log_file) - html_line += ''%log_file_base+display_column+'' - else: - html_line += ''+display_column+'' - elif i == 2: - if len(column) > 7: - column = column[:7] - html_line += ''+column+'' - elif i == 3: - if meta_tasks[cycle][line_num][1] and len(metatasks_state_string_cycle[cycle][ columns[1] ].split())!=1 and metatasks_state_cycle[cycle][columns[1]]: - column = metatasks_state_string_cycle[cycle][ columns[1] ] - if len(column)>15: - if column.split()[1] == 'SUCCEEDED': - html_line += ''+column[:15]+'' - elif column.split()[1] == 'QUEUED': - html_line += ''+column[:15]+'' - elif column.split()[1] in('DEAD','FAILED'): - html_line += ''+column[:15]+'' - elif column.split()[1] == 'RUNNING': - html_line += ''+column[:15]+'' - else: - html_line += ''+column[:15]+'' - else: - if column.split()[1] == 'SUCCEEDED': - html_line += ''+column+'' - elif column.split()[1] == 'QUEUED': - html_line += ''+column+'' - elif column.split()[1] in('DEAD','FAILED'): - html_line += ''+column+'' - elif column.split()[1] == 'RUNNING': - html_line += ''+column+'' - else: - html_line += ''+column+'' - elif column in text_color: - if column == 'SUCCEEDED': - html_line += ''+column+'' - elif column == 'QUEUED': - html_line += ''+column+'' - elif column in('DEAD','FAILED'): - html_line += ''+column+'' - elif column == 'RUNNING': - html_line += ''+column+'' - else: - html_line += ''+column+'' - else: - html_line += ''+column+'' - else: - if len(column)<6: - html_line += ''+column+'' - else: - html_line += ''+column+'' - if not skip_task: - html_line += '\n' - html_ptr.write(html_line) - - update_metatask_state_status_message = False - - found_still_running = False - cycle += 1 - update_pad = True - for find_cycle in range(cycle,len(rocoto_stat)): - for lines in rocoto_stat[find_cycle]: - if 'RUNNING' in lines: - found_still_running = True - break - break - if get_number_of_stats >= 0: - found_still_running = True - if cycle < len(rocoto_stat) or found_still_running: - html_line = '\n' - html_line += '\n
\n\n' - html_line += html_header_line - html_ptr.write(html_line) - get_number_of_stats -= 1 - else: - html_line = '\n' - html_line += '\n' - html_line += '\n' - html_ptr.write(html_line) - html_ptr.close() - if html_output_firstpass: - for meta_cycle in range(0,len(rocoto_stat)): - for execute_task in metatasks_state_cycle[meta_cycle]: - metatasks_state_cycle[meta_cycle][execute_task] = False - html_output_file = shbackslash( html_output_dir+'/index_exp.html' ) - html_ptr = open(html_output_file,'w') - html_ptr.write(ccs_html) - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - html_discribe_line = '\n\n\n\n'%(stat_update_time,PSLOT) - html_discribe_line += '\n\n
CollapseRefreshed: %sPSLOT: %s
ROTDIR: %sTurn Around Times
\n
\n'%(workflow_name,ROTDIR,PSLOT) - html_discribe_line += html_header_line - html_ptr.write( html_discribe_line ) - html_output_firstpass = False - #cycle = html_start_cycle - if not html_output_firstpass: - if send_html_to_rzdm: - print 'sending html files to rzdm using rsync ...' - cmd=batchexe('rsync')['-avzr','--delete', html_output_dir, rzdm_path] - stat=runstr(cmd) - if stat is None: - print 'warning rsync to %s failed'%html_output_dir - sys.exit(-1) - else: - print 'done' - sys.exit(0) - else: - - # Main Curses Screen Loop - # Write to curses screen when HTML is not outputted - highlight_CYCLE = False - highlight_WORKFLOW = False - get_execute_task_track = False - screen.clear() - global screen_resized - selected_tasks = collections.defaultdict(list) - selected_meta_tasks = collections.defaultdict(list) - execute_metatask = None - colapsed_metatask = None - task = 0 - while True: - if not check_file(workflow_file) or not check_file(database_file): - curses.endwin() - print;print - print 'rocoto_viwer quit because the Rocoto database or XML file used by this session when missing' - sys.exit(-1) - job_id = None - curses.noecho() - num_columns = default_column_length - if header is None: - header = ' ' - if highlight_WORKFLOW: - header_split = header.split('\n') - screen.addstr(0,0,header_split[0]+'\n') - screen.addstr(header_split[1],curses.A_STANDOUT) - else: - screen.addstr(0,0,header) - if update_pad is True: - num_lines = len(rocoto_stat[cycle]) - line_correction = 0 - for count_meta_tasks in meta_tasks[cycle]: - if count_meta_tasks[1] and metatasks_state_cycle[cycle][ count_meta_tasks[0] ]: - line_correction += len(count_meta_tasks[2]) - 1 - num_lines -= line_correction - update_pad = False - if mlines > num_lines: - pad = curses.newpad(mlines ,num_columns) - else: - pad = curses.newpad(num_lines+1 ,num_columns) - line_number = -1 - for line_num,line in enumerate(rocoto_stat[cycle]): - #debug.write('DISPLAY LINE: '+line+'\n') - colapsed_metatask = False - columns = line.split() - count_columns = line.split(' ') - spaces = [] - for c,sub_group in groupby(count_columns): - if c != '': continue - spaces.append(' '*len(list(sub_group))) - spaces.append('') - text_color = {'SUCCEEDED':3,'QUEUED':4,'DEAD':2,'FAILED':2,'RUNNING':6} - skip_task = False - - if not meta_tasks[cycle][line_num][1] and metatasks_state_cycle[cycle][ meta_tasks[cycle][line_num][2] ] : - skip_task = True - else: - line_number +=1 - if use_performance_metrics and len(columns) == 7: - for i in range(0,4): - columns.append('-') - for i,column in enumerate(columns): - if skip_task: continue - if not use_performance_metrics and i > 7: continue - execute_cycle = columns[0] - if i == 0: - if meta_tasks[cycle][line_num][1]: - if metatasks_state_cycle[cycle][columns[1]]: - if highlight_CYCLE: - pad.addstr(column, curses.A_STANDOUT) - else: - pad.addstr(column) - pad.addstr(' < ') - colapsed_metatask = True - if update_metatask_state_status_message or len(metatasks_state_string_cycle[cycle][ columns[1] ])==0: - get_state_list = [] - total_numer_of_tasks = len(meta_tasks[cycle][line_num][2]) - for check_metatask_line in rocoto_stat[cycle]: - split_check_metatask_line = check_metatask_line.split() - for each_metatask in meta_tasks[cycle][line_num][2]: - if each_metatask == split_check_metatask_line[1]: - get_state_list.append(split_check_metatask_line[3]) - red_override = False - metatask_state = columns[3] - if 'SUCCEEDED' in get_state_list: - metatask_state = '(%d/%d) SUCCEEDED'%(get_state_list.count('SUCCEEDED'),total_numer_of_tasks) - if 'QUEUED' in get_state_list: - metatask_state = '(%d/%d) QUEUED'%(get_state_list.count('QUEUED'),total_numer_of_tasks) - if 'RUNNING' in get_state_list: - metatask_state = '(%d/%d) RUNNING'%(get_state_list.count('RUNNING'),total_numer_of_tasks) - if 'FAILED' in get_state_list: - metatask_state = '(%d/%d) FAILED'%(get_state_list.count('FAILED'),total_numer_of_tasks) - red_override = True - if 'DEAD' in get_state_list: - red_override = True - metatask_state = '(%d/%d) DEAD'%(get_state_list.count('DEAD'),total_numer_of_tasks) - metatasks_state_string_cycle[cycle][ columns[1] ] = metatask_state - else: - if highlight_CYCLE: - pad.addstr(column, curses.A_STANDOUT) - else: - pad.addstr(column) - pad.addstr(' > ') - else: - if highlight_CYCLE: - pad.addstr(column,curses.A_STANDOUT) - pad.addstr(' ') - else: - pad.addstr(column+' ') - elif i == 1: - save_column = column - if colapsed_metatask: - column = metatask_name[column] - display_column = (column if len(column) < 19 else column[:19]) - if line_number == task and not highlight_CYCLE and not highlight_WORKFLOW : - pad.addstr(display_column,curses.A_STANDOUT) - execute_task_track = save_column - if colapsed_metatask: - execute_metatask_check = True - execute_metatask = column - metatask_list_of_selected_metatask = meta_tasks[cycle][line_num][2] - else: - execute_metatask_check = False - execute_metatask = None - metatask_list_of_selected_metatask = None - execute_task = column - else: - #if column in metatask_list_by_name[metatask_name[column]]: - # display_column = ' '+display_column - if column in selected_tasks[execute_cycle]: - pad.addstr(display_column, highlightSelectedText ) - elif column in selected_meta_tasks[execute_cycle]: - pad.addstr(display_column, highlightSelectedText ) - else: - pad.addstr(display_column) - pad.addstr(' '*(21-len(display_column))) - elif i == 2: - job_id = column.strip() - if len(job_id) > 9: - job_id = job_id[:9] - if job_id == '-': - pad.addstr(job_id+' '*9) - else: - pad.addstr(job_id+' '*(10-len(job_id))) - elif i == 3: - if meta_tasks[cycle][line_num][1] and len(metatasks_state_string_cycle[cycle][ columns[1] ].split())!=1 and metatasks_state_cycle[cycle][columns[1]]: - column = metatasks_state_string_cycle[cycle][ columns[1] ] - if red_override: - the_text_color = 2 - else: - the_text_color = text_color[column.split()[1]] - - if len(column) >= 15: - pad.addstr( column[:15],curses.color_pair(the_text_color)|curses.A_STANDOUT) - column = column[:15] - else: - pad.addstr( column,curses.color_pair(the_text_color)|curses.A_STANDOUT) - elif column in text_color: - pad.addstr(column, curses.color_pair(text_color[column])|curses.A_STANDOUT) - else: - pad.addstr(column) - pad.addstr(' '*(16-len(column)),curses.color_pair(8)) - elif i in (4,5,6,7,8,9,10): - if len(column) < 8: - pad.addstr(column+' '*(8-len(column))) - else: - pad.addstr(column.strip()+' ') - - if not skip_task: - pad.addstr('\n') - - update_metatask_state_status_message = False - pad.refresh( pad_pos, 0, 2,0, mlines-4,mcols) - - entire_workflow = 'Hit to open cycle based information page (implementation pending)' - entire_cycle = '********* The ENTIRE CYCLE has been selected for an action **********' - - try: - if highlight_WORKFLOW: - screen.addstr(mlines-2,0,entire_workflow,curses.A_BOLD) - else: - screen.addstr(mlines-2,0,' '*len(entire_workflow)) - if highlight_CYCLE: - screen.addstr(mlines-2,0,entire_cycle,curses.A_BOLD) - elif not highlight_WORKFLOW: - screen.addstr(mlines-2,0,' '*len(entire_cycle)) - if pad_pos < num_lines-mlines+4 or pad_pos > 0: - screen.addstr(mlines-1,0,' '*len(bottom_message_scroll)) - screen.addstr(mlines-1,0,bottom_message_scroll,curses.A_BOLD) - else: - screen.addstr(mlines-1,0,' '*len(bottom_message_scroll)) - screen.addstr(mlines-1,0,bottom_message,curses.A_BOLD) - except: - std_time.sleep(1) - pass - - if num_columns > mcols: - curses.endwin() - print - print 'Your terminal is only %s characters must be at least %s to display workflow status'%(str(mcols),str(num_columns)) - sys.exit(-1) - - if loading_stat: - dot_stat = (0 if dot_stat == len(dots)-1 else dot_stat+1 ) - screen.addstr(mlines-2,0,'Running rocotostat ') - screen.addstr(mlines-2,20,dots[dot_stat]) - try: - rocoto_stat_tmp = queue_stat.get_nowait() - except: - rocoto_stat_tmp = '' - if len(rocoto_stat_tmp) != 0: - (rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles) = rocoto_stat_tmp - process_get_rocoto_stat.join() - process_get_rocoto_stat.terminate() - update_pad = True - loading_stat = False - rocoto_stat_tmp = '' - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - header = header_string - header = header.replace('t'*16,stat_update_time) - header = header.replace('pslot',PSLOT) - reduce_header_size = int((len(PSLOT)-len('PSLOT'))/2) - if reduce_header_size > 0: - header = header[:-reduce_header_size] - header = header[reduce_header_size:] - screen.addstr(mlines-2,0,'Updated new rocotostatus: %s'%stat_update_time+' '*48) - screen.refresh() - std_time.sleep(0.5) - screen.addstr(mlines-2,0,' '*100) - screen.refresh() - - if loading_check: - if time() - current_check_time > 5: - dot_check = (0 if dot_check == len(dots)-1 else dot_check+1 ) - loc = (0 if not loading_stat else 27) - screen.addstr(mlines-2,loc,'Running rocotocheck ') - screen.addstr(mlines-2,loc+20,dots[dot_check]) - try: - rocoto_check = queue_check.get_nowait() - except: - pass - if len(rocoto_check) != 0: - process_get_rocoto_check.join() - process_get_rocoto_check.terminate() - loading_check = False - if time() - current_check_time > 5: - event = screen.getch() - time_inc = 0.0 - while event != curses.KEY_ENTER and event != 10: - message_string = 'rocotocheck for %s %s is ready for vieweing'%(params_check[2],params_check[3]) - message_string = (message_string if len(message_string) < mcols else message_string[:mcols-1]) - time_inc += 1 - if time_inc > 4: - screen.addstr(mlines-2,0, message_string) - screen.addstr(mlines-2,len(message_string),' ') - time_inc = 0.0 - else: - screen.addstr(mlines-2,0,message_string) - screen.addstr(mlines-2,len(message_string),' ',curses.A_BOLD) - event = screen.getch() - display_results(rocoto_check,screen,params_check) - rocoto_check = '' - - curses.curs_set(0) - curses.halfdelay(2) - screen.keypad(1) - event = screen.getch() - - if event in (curses.KEY_LEFT, curses.KEY_RIGHT): - highlight_CYCLE = False - highlight_WORKFLOW = False - if event == curses.KEY_LEFT: - pad_pos = 0 - #debug.write('KEY_LEFT %s\n'%pad_pos) - if cycle - 1 >= 0: - cycle -= 1 - elif event == curses.KEY_RIGHT: - pad_pos = 0 - #debug.write('KEY_RIGHT %s\n'%pad_pos) - if cycle + 1 < num_cycle: - cycle += 1 - num_lines = len(rocoto_stat[cycle]) - line_correction = 0 - for count_meta_tasks in meta_tasks[cycle]: - if count_meta_tasks[1] and metatasks_state_cycle[cycle][ count_meta_tasks[0] ]: - line_correction += len(count_meta_tasks[2])-1 - num_lines -= line_correction - if task > num_lines-1: - task = num_lines-1 - update_pad = True - if event == ord('Q'): - break - if get_execute_task_track: - get_execute_task_track = False - if execute_task_track in metatasks_state_cycle[cycle]: - metatasks_state_cycle[cycle][execute_task_track] = not metatasks_state_cycle[cycle][execute_task_track] - update_metatask_state_status_message = True - update_pad = True - if event == curses.KEY_MOUSE: - mouse_id, mouse_x, mouse_y, mouse_z, button_state = curses.getmouse() - task_mouse_pos = pad_pos+mouse_y-2 - if task_mouse_pos >= 0 and task_mouse_pos < num_lines: - task = task_mouse_pos - update_pad = True - if button_state & curses.BUTTON1_DOUBLE_CLICKED and mouse_x in range(12,15): - get_execute_task_track = True - if event == ord('x'): - if execute_task_track in metatasks_state_cycle[cycle]: - metatasks_state_cycle[cycle][execute_task_track] = not metatasks_state_cycle[cycle][execute_task_track] - update_metatask_state_status_message = True - update_pad = True - if screen_resized: - screen.erase() - screen.refresh() - update_pad = True - task = pad_pos - screen_resized = False - curses.resizeterm(mlines, mcols) - #debug.write('SCREEN RESIZED %s (%d,%d)\n'%(pad_pos,mlines,mcols)) - if mcols < default_column_length: - curses.endwin() - print - print 'Your terminal is only %d characters must be at least %d to display workflow status'%(mcols,default_column_length) - sys.exit(-1) - elif event in ( curses.KEY_NPAGE, ord('d') ): - highlight_CYCLE = False - highlight_WORKFLOW = False - if pad_pos + mlines < num_lines-mlines+5: - pad_pos += mlines - 5 - task += mlines - 5 - else: - pad_pos = num_lines-mlines+5 - task = num_lines-1 - update_pad = True - elif event in ( curses.KEY_PPAGE, ord('u') ): - highlight_CYCLE = False - highlight_WORKFLOW = False - if pad_pos != 0: - if pad_pos - mlines > 0: - pad_pos -= mlines - 5 - if task > pad_pos+mlines-6: - task -= mlines - 5 - else: - pad_pos = 0 - task = 0 - update_pad = True - elif event in (curses.KEY_UP, curses.KEY_SR): - if task == 0: - if highlight_CYCLE: - highlight_CYCLE = False - highlight_WORKFLOW = True - if not highlight_WORKFLOW: - highlight_CYCLE = True - if task != pad_pos: - update_pad = True - task -= 1 - elif pad_pos != 0: - pad_pos -= 1 - task -= 1 - if event == curses.KEY_SR: - if execute_metatask_check: - if execute_metatask in selected_meta_tasks[execute_cycle]: - if len(selected_meta_tasks[execute_cycle]) > 0: - selected_meta_tasks[execute_cycle].remove(execute_metatask) - else: - selected_meta_tasks[execute_cycle].append(execute_metatask) - else: - if execute_task in selected_tasks[execute_cycle]: - if len(selected_tasks[execute_cycle]) > 0: - selected_tasks[execute_cycle].remove(execute_task) - else: - selected_tasks[execute_cycle].append(execute_task) - update_pad = True - elif event in ( curses.KEY_DOWN, curses.KEY_SF ): - if highlight_CYCLE or highlight_WORKFLOW: - task = -1 - highlight_CYCLE = False - highlight_WORKFLOW = False - if task != num_lines-1 and task < pad_pos+mlines-6: - task += 1 - elif pad_pos < num_lines-mlines+5: - pad_pos += 1 - task += 1 - if event == curses.KEY_SF: - if execute_metatask_check: - if execute_metatask in selected_meta_tasks[execute_cycle]: - if len(selected_meta_tasks[execute_cycle]): - selected_meta_tasks[execute_cycle].remove(execute_metatask) - else: - selected_meta_tasks[execute_cycle].append(execute_metatask) - else: - if execute_task in selected_tasks[execute_cycle]: - if len(selected_tasks[execute_cycle]) > 0: - selected_tasks[execute_cycle].remove(execute_task) - else: - selected_tasks[execute_cycle].append(execute_task) - update_pad = True - elif event == ord('c'): - if loading_check == True: - screen.addstr(mlines-2,0,'rocotocheck is all reading running ') - screen.refresh() - std_time.sleep(0.5) - screen.addstr(mlines-2,0,' '*100) - screen.refresh() - else: - loc = (0 if not loading_stat else 27) - screen.addstr(mlines-2,loc,'Running rocotocheck ') - screen.refresh() - params_check = (workflow_file, database_file, execute_task, execute_cycle, 'check') - process_get_rocoto_check = Process( target=get_rocoto_check, args=[params_check, queue_check] ) - process_get_rocoto_check.start() - loading_check = True - current_check_time = time() - elif event == ord('f'): - log_file = '' - for find_task in tasks_ordered: - if find_task[0] == execute_task: - log_file = find_task[2].replace('CYCLE', execute_cycle[:-2] ) - if check_file(log_file): - links = [] - links.append(log_file) - try: - make_symlinks_in(links,EXPDIR,force=True) - except: - pass - elif event in (curses.KEY_ENTER, 10, 13): - - if execute_metatask_check: - selected_tasks[execute_cycle] = list_selector( screen, selected_tasks[execute_cycle], metatask_list_of_selected_metatask ) - screen.erase() - else: - if execute_task in selected_tasks[execute_cycle]: - if len(selected_tasks[execute_cycle]) > 0: - selected_tasks[execute_cycle].remove(execute_task) - else: - selected_tasks[execute_cycle].append(execute_task) - - elif event == ord('r'): - screen.clear() - process = '' - if highlight_CYCLE: - screen.addstr('Are you sure you want to rewind all the tasks in the cycle %s by running:\n\n'%execute_cycle) - process = '-a' - #highlight_WORKFLOW = False - elif execute_metatask_check and len(selected_tasks[execute_cycle]) == 0: - for tasks in metatask_list_of_selected_metatask: - process += '-t ' + tasks+' ' - screen.addstr('Are you sure you want to rewind all the tasks in the metatask (%s) by running:\n\n'%execute_task) - elif len(selected_tasks[execute_cycle]) != 0 or len(selected_meta_tasks[execute_cycle]) != 0: - if len(selected_tasks[execute_cycle]) != 0: - selected_tasks_string = '' - screen.addstr('Selected tasks:\n\n') - for tasks in selected_tasks[execute_cycle]: - selected_tasks_string += tasks+'\t' - process += '-t ' + tasks+' ' - screen.addstr(selected_tasks_string+'\n\n') - if len(selected_meta_tasks[execute_cycle]) != 0: - selected_tasks_string = '' - screen.addstr('Selected %d entire meta-tasks and their tasks:\n\n'%len( selected_meta_tasks[execute_cycle])) - for meta_task_selected in selected_meta_tasks[execute_cycle]: - for tasks in metatask_list_by_name[meta_task_selected]: - selected_tasks_string += tasks+'\t' - process += '-t ' + tasks+' ' - screen.addstr(selected_tasks_string+'\n\n') - screen.addstr('\nAre you sure you want to rewind all these seleted tasks by running:\n\n') - elif len(selected_tasks[execute_cycle]) == 0: - process = '-t '+ execute_task - screen.addstr('Are you sure you want to rewind the single task %s by running:\n\n'%execute_task) - screen.addstr('rocotorewind -c %s -d %s -w %s %s\n\n'%(execute_cycle,basename(database_file),basename(workflow_file),process)) - screen.addstr('Enter: es or o',curses.A_BOLD) - while True: - event = screen.getch() - if event == ord('y') or event == ord('Y'): - params = (workflow_file, database_file, execute_cycle,process) - results = rocoto_rewind(params) - results_params = ('','','rewind',execute_cycle,'tasks') - try: - display_results(results,screen,results_params) - except: - screen.addstr('\n\nRewind of this job was successful but displaying of the stdout failed\n') - screen.addstr('Output has been written out to the file rocotorewind_output.log\n') - screen.addstr('Press to continue') - with open('rocotorewind_output.log','a') as rocotorewind_logfile: - rocotorewind_logfile.write('\n\n'+results) - while True: - event = screen.getch() - if event in (curses.KEY_ENTER, 10, 13): - break - selected_tasks[execute_cycle] = [] - break - elif event == ord('n') or event == ord('N'): - break - screen.clear() - update_pad = True - elif event == ord('U'): - selected_tasks[execute_cycle] = [] - selected_meta_tasks[execute_cycle] = [] - update_pad = True - elif event == ord('b'): - process = '' - screen.clear() - list_meta_tasks = '' - list_of_tasks = '' - boot_task_list = '' ; tasks_to_boot = [] - boot_metatask_list = '' ; metatasks_to_boot = [] - if highlight_CYCLE: - screen.addstr('You have selected to boot the entire cycle %s:\n\n'%execute_cycle,curses.A_BOLD) - metatasks_to_boot = metatask_list_per_cycle[cycle] - tasks_to_boot = tasks_in_cycle[cycle] - elif len(selected_tasks[execute_cycle]) != 0 or len(selected_meta_tasks[execute_cycle]) != 0: - screen.addstr('You have a list selected tasks and/or metatasks to boot:\n\n',curses.A_BOLD) - metatasks_to_boot = selected_tasks[execute_cycle] - tasks_to_boot = selected_tasks[execute_cycle] - elif execute_metatask_check: - screen.addstr('Are you sure you want boot the entire meta task %s by running:\n\n'%execute_metatask) - metatasks_to_boot.append(execute_metatask) - elif len(selected_tasks[execute_cycle]) == 0: - tasks_to_boot.append(execute_task) - screen.addstr('Are you sure you want boot the task %s by running rocotoboot with:'%execute_task) - else: - update_pad = True - continue - - if len(metatasks_to_boot) > 0: - list_meta_tasks = ' ' - screen.addstr('Metatasks selected in cycle:\n\n',curses.A_BOLD) - for meta_task in metatasks_to_boot: - list_meta_tasks += meta_task+' ' - boot_metatask_list += meta_task+',' - boot_metatask_list = boot_metatask_list[:-1] - screen.addstr( list_meta_tasks ) - if len(tasks_to_boot) > 0: - list_of_tasks = ' ' - screen.addstr('\n\nTasks selected in cycle:\n\n',curses.A_BOLD) - for a_task in tasks_to_boot: - list_of_tasks += a_task+' ' - boot_task_list += a_task+',' - boot_task_list = boot_task_list[:-1] - screen.addstr( list_of_tasks ) - - screen.addstr('\n\nAre you sure you want to boot all the tasks and/or metatasks in the cycle %s by running:\n\n'%execute_cycle,curses.A_BOLD) - if len(boot_metatask_list) != 0: - list_meta_tasks = '--metatasks '+"'"+boot_metatask_list+"'" - if len(boot_task_list) != 0: - list_of_tasks = ' --tasks '+"'"+boot_task_list+"'" - screen.addstr('rocotoboot -d %s -w %s %s\n\n'%(basename(database_file),basename(workflow_file),list_meta_tasks+list_of_tasks)) - screen.addstr('Enter: es or o',curses.A_BOLD) - - while True: - event = screen.getch() - if event == ord('y') or event == ord('Y'): - params = (workflow_file, database_file, execute_cycle, boot_metatask_list, boot_task_list) - results = rocoto_boot(params) - display_results(results,screen,('','',execute_cycle,'rocotoboot_output')) - break - elif event == ord('n') or event == ord('N'): - break - screen.clear() - update_pad = True - elif event == ord('R'): - screen.addstr(mlines-2,0,'Running rocotorun and rocotostat ...'+' '*60,curses.A_BOLD) - params = (workflow_file, database_file) - rocoto_run(params) - update_pad = True - screen.clear() - if loading_stat == True: - screen.addstr(mlines-2,0,'rocotostat is all reading running'+' '*60) - screen.refresh() - std_time.sleep(0.5) - else: - start_time = 0 - elif event == ord('/'): - curses.echo() - find_next = 1 - screen.addstr(mlines-3,0,' '*100) - screen.refresh() - screen.addstr(mlines-3,0,'/') - screen.refresh() - search_string = screen.getstr(mlines-3,1,50) - break_twice = False - screen.addstr(mlines-3,0,' '*100) - screen.refresh() - for every_cycle in range(0,len(rocoto_stat)): - for line_number,line in enumerate(rocoto_stat[every_cycle]): - if search_string in line: - task = line_number - if num_lines < mlines: - pad_pos = 0 - else: - pad_pos = task - update_pad = True - cycle = every_cycle - break_twice = True - break - if break_twice: - screen.clear() - break - else: - find_next = 1 - elif (event == ord('n') or event == ord('N')) and len(search_string) != 0: - if event == ord('n'): - find_next += 1 - else: - if find_next - 1 >= 1: - find_next -= 1 - found_next = 0 - break_twice = False - for every_cycle in range(0,len(rocoto_stat)): - for line_number,line in enumerate(rocoto_stat[every_cycle]): - if search_string in line: - found_next += 1 - if find_next == found_next: - task = line_number - if num_lines < mlines: - pad_pos = 0 - else: - pad_pos = task - update_pad = True - cycle = every_cycle - break_twice = True - break - if break_twice: - screen.clear() - break - if not break_twice: - find_next = 1 - - elif event == ord('F'): - for find_cycle in range(0,len(rocoto_stat)): - for lines in rocoto_stat[find_cycle]: - if 'RUNNING' in line: - break - break - if find_cycle > 1: - cycle = find_cycle - 2 - update_pad = True - elif event == ord('l'): - start_time -= stat_read_time_delay - elif event == ord('h'): - update_pad = True - help_screen(screen) - screen.clear() - current_time = time() - diff = current_time - start_time - if diff > stat_read_time_delay and not loading_stat: - start_time = current_time - if not use_multiprocessing: - params = (workflow_file, database_file, tasks_ordered, metatask_list,cycledef_group_cycles) - (rocoto_stat, tasks_ordered, metatask_list,cycledef_group_cycles) = get_rocoto_stat( params, Queue() ) - stat_update_time = str(datetime.datetime.now()).rsplit(':',1)[0] - header = header_string - header = header.replace('t'*16,stat_update_time) - header = header.replace('pslot',PSLOT) - reduce_header_size = int((len(PSLOT)-len('PSLOT'))/2) - if reduce_header_size > 0: - header = header[:-reduce_header_size] - header = header[reduce_header_size:] - update_pad = True - screen.clear() - else: - loading_stat = True - screen.addstr(mlines-2,0,'Running rocotostat ') - params = (workflow_file, database_file, tasks_ordered, metatask_list,cycledef_group_cycles) - process_get_rocoto_stat = Process( target=get_rocoto_stat, args=[params, queue_stat] ) - process_get_rocoto_stat.start() - - if use_multiprocessing: - if process_get_rocoto_stat is not None: - if process_get_rocoto_stat.is_alive(): - process_get_rocoto_stat.terminate() - if process_get_rocoto_check is not None: - if process_get_rocoto_check.is_alive(): - process_get_rocoto_check.terminate() - - #debug.close() - -if __name__ == '__main__': - if not load_produtil_pythonpath(): - print '\n\nCRITICAL ERROR: The produtil package could not be loaded from your system' - sys.exit(-1) - from produtil.fileop import remove_file - try: - signal.signal(signal.SIGWINCH, sigwinch_handler) - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - if sys.stdin.isatty(): - curses.wrapper(main) - else: - screen = 'dummy' - main(screen) - remove_file(temp_workflow_file) - except KeyboardInterrupt: - print "Got KeyboardInterrupt exception. Exiting..." - sys.exit(-1) diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py deleted file mode 100755 index 05fedec7ca..0000000000 --- a/ush/rocoto/setup_expt.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### - -import os -import sys -import glob -import shutil -from datetime import datetime -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter - - -global machines -global expdir, configdir, comrot, pslot, resdet, resens, nens, cdump, idate, edate, gfs_cyc - - -machines = ['THEIA', 'WCOSS_C', 'WCOSS_DELL_P3'] - - -def makedirs_if_missing(d): - if not os.path.exists(d): - os.makedirs(d) - - -def create_EXPDIR(): - - makedirs_if_missing(expdir) - configs = glob.glob('%s/config.*' % configdir) - if len(configs) == 0: - msg = 'no config files found in %s' % configdir - raise IOError(msg) - for config in configs: - shutil.copy(config, expdir) - - return - - -def create_COMROT(): - - idatestr = idate.strftime('%Y%m%d%H') - cymd = idate.strftime('%Y%m%d') - chh = idate.strftime('%H') - - makedirs_if_missing(comrot) - - # Link ensemble member initial conditions - enkfdir = 'enkf.%s.%s/%s' % (cdump, cymd, chh) - makedirs_if_missing(os.path.join(comrot, enkfdir)) - for i in range(1, nens + 1): - makedirs_if_missing(os.path.join(comrot, enkfdir, 'mem%03d' % i)) - os.symlink(os.path.join(icsdir, idatestr, 'C%d' % resens, 'mem%03d' % i, 'INPUT'), - os.path.join(comrot, enkfdir, 'mem%03d' % i, 'INPUT')) - - # Link deterministic initial conditions - detdir = '%s.%s/%s' % (cdump, cymd, chh) - makedirs_if_missing(os.path.join(comrot, detdir)) - os.symlink(os.path.join(icsdir, idatestr, 'C%d' % resdet, 'control', 'INPUT'), - os.path.join(comrot, detdir, 'INPUT')) - - # Link bias correction and radiance diagnostics files - for fname in ['abias', 'abias_pc', 'abias_air', 'radstat']: - os.symlink(os.path.join(icsdir, idatestr, '%s.t%sz.%s' % (cdump, chh, fname)), - os.path.join(comrot, detdir, '%s.t%sz.%s' % (cdump, chh, fname))) - - return - - -def edit_baseconfig(): - - base_config = '%s/config.base' % expdir - - here = os.path.dirname(__file__) - top = os.path.abspath(os.path.join( - os.path.abspath(here), '../..')) - - # make a copy of the default before editing - shutil.copy(base_config, base_config + '.default') - - print '\nSDATE = %s\nEDATE = %s' % (idate, edate) - with open(base_config + '.default', 'rt') as fi: - with open(base_config + '.new', 'wt') as fo: - for line in fi: - line = line.replace('@MACHINE@', machine.upper()) \ - .replace('@PSLOT@', pslot) \ - .replace('@SDATE@', idate.strftime('%Y%m%d%H')) \ - .replace('@EDATE@', edate.strftime('%Y%m%d%H')) \ - .replace('@CASEENS@', 'C%d' % resens) \ - .replace('@CASECTL@', 'C%d' % resdet) \ - .replace('@NMEM_ENKF@', '%d' % nens) \ - .replace('@HOMEgfs@', top) \ - .replace('@gfs_cyc@', '%d' % gfs_cyc) - if expdir is not None: - line = line.replace('@EXPDIR@', os.path.dirname(expdir)) - if comrot is not None: - line = line.replace('@ROTDIR@', os.path.dirname(comrot)) - if 'ICSDIR' in line: - continue - fo.write(line) - os.unlink(base_config) - os.rename(base_config + '.new', base_config) - - print '' - print 'EDITED: %s/config.base as per user input.' % expdir - print 'DEFAULT: %s/config.base.default is for reference only.' % expdir - print 'Please verify and delete the default file before proceeding.' - print '' - - return - - -if __name__ == '__main__': - - description = '''Setup files and directories to start a GFS parallel. -Create EXPDIR, copy config files -Create COMROT experiment directory structure, -link initial condition files from $ICSDIR to $COMROT''' - - parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') - parser.add_argument('--resdet', help='resolution of the deterministic model forecast', type=int, required=False, default=384) - parser.add_argument('--resens', help='resolution of the ensemble model forecast', type=int, required=False, default=192) - parser.add_argument('--comrot', help='full path to COMROT', type=str, required=False, default=None) - parser.add_argument('--expdir', help='full path to EXPDIR', type=str, required=False, default=None) - parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', type=str, required=True) - parser.add_argument('--edate', help='end date experiment', type=str, required=True) - parser.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=True) - parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None) - parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20) - parser.add_argument('--cdump', help='CDUMP to start the experiment', type=str, required=False, default='gdas') - parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False) - - args = parser.parse_args() - - if os.path.exists('/scratch3'): - machine = 'THEIA' - elif os.path.exists('/gpfs') and os.path.exists('/etc/SuSE-release'): - machine = 'WCOSS_C' - elif os.path.exists('/gpfs/dell2'): - machine = 'WCOSS_DELL_P3' - else: - print 'workflow is currently only supported on: %s' % ' '.join(machines) - raise NotImplementedError('Cannot auto-detect platform, ABORT!') - - configdir = args.configdir - if not configdir: - configdir = os.path.abspath(os.path.dirname(__file__) + '/../parm/config') - - pslot = args.pslot - idate = datetime.strptime(args.idate, '%Y%m%d%H') - edate = datetime.strptime(args.edate, '%Y%m%d%H') - icsdir = args.icsdir - resdet = args.resdet - resens = args.resens - comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot) - expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot) - nens = args.nens - cdump = args.cdump - gfs_cyc = args.gfs_cyc - - if not os.path.exists(icsdir): - msg = 'Initial conditions do not exist in %s' % icsdir - raise IOError(msg) - - # COMROT directory - create_comrot = True - if os.path.exists(comrot): - print - print 'COMROT already exists in %s' % comrot - print - overwrite_comrot = raw_input('Do you wish to over-write COMROT [y/N]: ') - create_comrot = True if overwrite_comrot in ['y', 'yes', 'Y', 'YES'] else False - if create_comrot: - shutil.rmtree(comrot) - - if create_comrot: - create_COMROT() - - # EXP directory - create_expdir = True - if os.path.exists(expdir): - print - print 'EXPDIR already exists in %s' % expdir - print - overwrite_expdir = raw_input('Do you wish to over-write EXPDIR [y/N]: ') - create_expdir = True if overwrite_expdir in ['y', 'yes', 'Y', 'YES'] else False - if create_expdir: - shutil.rmtree(expdir) - - if create_expdir: - create_EXPDIR() - edit_baseconfig() - - sys.exit(0) diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py deleted file mode 100755 index 08d6c65956..0000000000 --- a/ush/rocoto/setup_expt_fcstonly.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### - -import os -import sys -import glob -import shutil -from datetime import datetime -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter - - -global machines -global expdir, configdir, comrot, pslot, res, idate, edate, gfs_cyc - - -machines = ['THEIA', 'WCOSS_C', 'WCOSS_DELL_P3'] - - -def makedirs_if_missing(d): - if not os.path.exists(d): - os.makedirs(d) - - -def create_EXPDIR(): - - makedirs_if_missing(expdir) - configs = glob.glob('%s/config.*' % configdir) - if len(configs) == 0: - msg = 'no config files found in %s' % configdir - raise IOError(msg) - for config in configs: - shutil.copy(config, expdir) - - return - - -def create_COMROT(): - - makedirs_if_missing(comrot) - - return - - -def edit_baseconfig(): - - base_config = '%s/config.base' % expdir - - here = os.path.dirname(__file__) - top = os.path.abspath(os.path.join(os.path.abspath(here), '../..')) - - # make a copy of the default before editing - shutil.copy(base_config, base_config + '.default') - - print '\nSDATE = %s\nEDATE = %s' % (idate, edate) - with open(base_config + '.default', 'rt') as fi: - with open(base_config + '.new', 'wt') as fo: - for line in fi: - line = line.replace('@MACHINE@', machine.upper()) \ - .replace('@PSLOT@', pslot) \ - .replace('@SDATE@', idate.strftime('%Y%m%d%H')) \ - .replace('@EDATE@', edate.strftime('%Y%m%d%H')) \ - .replace('@CASECTL@', 'C%d' % res) \ - .replace('@HOMEgfs@', top) \ - .replace('@gfs_cyc@', '%d' % gfs_cyc) - if expdir is not None: - line = line.replace('@EXPDIR@', os.path.dirname(expdir)) - if comrot is not None: - line = line.replace('@ROTDIR@', os.path.dirname(comrot)) - line = line.replace('@ICSDIR@', os.path.join(os.path.dirname(comrot), 'FV3ICS')) - fo.write(line) - os.unlink(base_config) - os.rename(base_config + '.new', base_config) - - print '' - print 'EDITED: %s/config.base as per user input.' % expdir - print 'DEFAULT: %s/config.base.default is for reference only.' % expdir - print 'Please verify and delete the default file before proceeding.' - print '' - - return - - -if __name__ == '__main__': - - description = '''Setup files and directories to start a GFS parallel. -Create EXPDIR, copy config files -Create COMROT experiment directory structure''' - - parser = ArgumentParser(description=description, formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') - parser.add_argument('--res', help='resolution of the model forecast', type=int, required=False, default=192) - parser.add_argument('--comrot', help='full path to COMROT', type=str, required=False, default=None) - parser.add_argument('--expdir', help='full path to EXPDIR', type=str, required=False, default=None) - parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', type=str, required=True) - parser.add_argument('--edate', help='end date experiment', type=str, required=True) - parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None) - parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False) - - args = parser.parse_args() - - if os.path.exists('/scratch3'): - machine = 'THEIA' - elif os.path.exists('/gpfs') and os.path.exists('/etc/SuSE-release'): - machine = 'WCOSS_C' - elif os.path.exists('/gpfs/dell2'): - machine = 'WCOSS_DELL_P3' - else: - print 'workflow is currently only supported on: %s' % ' '.join(machines) - raise NotImplementedError('Cannot auto-detect platform, ABORT!') - - configdir = args.configdir - if not configdir: - configdir = os.path.abspath(os.path.dirname(__file__) + '/../parm/config') - - pslot = args.pslot - idate = datetime.strptime(args.idate, '%Y%m%d%H') - edate = datetime.strptime(args.edate, '%Y%m%d%H') - res = args.res - comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot) - expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot) - gfs_cyc = args.gfs_cyc - - # COMROT directory - create_comrot = True - if os.path.exists(comrot): - print - print 'COMROT already exists in %s' % comrot - print - overwrite_comrot = raw_input('Do you wish to over-write COMROT [y/N]: ') - create_comrot = True if overwrite_comrot in ['y', 'yes', 'Y', 'YES'] else False - if create_comrot: - shutil.rmtree(comrot) - - if create_comrot: - create_COMROT() - - # EXP directory - create_expdir = True - if os.path.exists(expdir): - print - print 'EXPDIR already exists in %s' % expdir - print - overwrite_expdir = raw_input('Do you wish to over-write EXPDIR [y/N]: ') - create_expdir = True if overwrite_expdir in ['y', 'yes', 'Y', 'YES'] else False - if create_expdir: - shutil.rmtree(expdir) - - if create_expdir: - create_EXPDIR() - edit_baseconfig() - - sys.exit(0) diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py deleted file mode 100755 index c296dbbac0..0000000000 --- a/ush/rocoto/setup_workflow.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### -''' - PROGRAM: - Create the ROCOTO workflow given the configuration of the GFS parallel - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov - - FILE DEPENDENCIES: - 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. - Without these dependencies, the script will fail - - OUTPUT: - 1. PSLOT.xml: XML workflow - 2. PSLOT.crontab: crontab for ROCOTO run command -''' - -import os -import sys -import numpy as np -from datetime import datetime, timedelta -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from collections import OrderedDict -import rocoto -import workflow_utils as wfu - - -def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a GFS parallel.', formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir', help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - args = parser.parse_args() - - configs = wfu.get_configs(args.expdir) - - _base = wfu.config_parser([wfu.find_config('config.base', configs)]) - - if not os.path.samefile(args.expdir, _base['EXPDIR']): - print 'MISMATCH in experiment directories!' - print 'config.base: EXPDIR = %s' % repr(_base['EXPDIR']) - print 'input arg: --expdir = %s' % repr(args.expdir) - sys.exit(1) - - gfs_steps = ['prep', 'anal', 'fcst', 'postsnd', 'post', 'awips', 'gempak', 'vrfy', 'arch'] - hyb_steps = ['eobs', 'eomg', 'eupd', 'ecen', 'efcs', 'epos', 'earc'] - - steps = gfs_steps + hyb_steps if _base.get('DOHYBVAR', 'NO') == 'YES' else gfs_steps - - dict_configs = wfu.source_configs(configs, steps) - - # Check and set gfs_cyc specific variables - if dict_configs['base']['gfs_cyc'] != 0: - dict_configs['base'] = get_gfs_cyc_dates(dict_configs['base']) - - # First create workflow XML - create_xml(dict_configs) - - # Next create the crontab - wfu.create_crontab(dict_configs['base']) - - return - - -def get_gfs_cyc_dates(base): - ''' - Generate GFS dates from experiment dates and gfs_cyc choice - ''' - - base_out = base.copy() - - gfs_cyc = base['gfs_cyc'] - sdate = base['SDATE'] - edate = base['EDATE'] - - interval_gfs = wfu.get_gfs_interval(gfs_cyc) - - # Set GFS cycling dates - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print 'W A R N I N G!' - print 'Starting date for GFS cycles is after Ending date of experiment' - print 'SDATE = %s, EDATE = %s' % (sdate.strftime('%Y%m%d%H'), edate.strftime('%Y%m%d%H')) - print 'SDATE_GFS = %s, EDATE_GFS = %s' % (sdate_gfs.strftime('%Y%m%d%H'), edate_gfs.strftime('%Y%m%d%H')) - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - fhmax_gfs = {} - for hh in ['00', '06', '12', '18']: - fhmax_gfs[hh] = base.get('FHMAX_GFS_%s' % hh, 'FHMAX_GFS_00') - base_out['FHMAX_GFS'] = fhmax_gfs - - return base_out - - -def get_preamble(): - ''' - Generate preamble for XML - ''' - - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['PSLOT']) - strings.append('\t\n' % base['SDATE'].strftime('%Y%m%d%H%M')) - strings.append('\t\n' % base['EDATE'].strftime('%Y%m%d%H%M')) - - if base['gfs_cyc'] != 0: - strings.append(get_gfs_dates(base)) - strings.append('\n') - - strings.append('\t\n') - strings.append('\t\n' % base['RUN_ENVIR']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['EXPDIR']) - strings.append('\t\n' % base['ROTDIR']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['HOMEgfs']) - strings.append('\t\n' % base['BASE_JOB']) - strings.append('\t\n' % base['DMPDIR']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['ACCOUNT']) - strings.append('\t\n' % base['QUEUE']) - strings.append('\t\n' % base['QUEUE_ARCH']) - strings.append('\t\n' % wfu.get_scheduler(base['machine'])) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_gfs_dates(base): - ''' - Generate GFS dates entities - ''' - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['SDATE_GFS'].strftime('%Y%m%d%H%M')) - strings.append('\t\n' % base['EDATE_GFS'].strftime('%Y%m%d%H%M')) - strings.append('\t\n' % base['INTERVAL_GFS']) - - return ''.join(strings) - - -def get_gdasgfs_resources(dict_configs, cdump='gdas'): - ''' - Create GDAS or GFS resource entities - ''' - - base = dict_configs['base'] - machine = base.get('machine', 'WCOSS_C') - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - - tasks = ['prep', 'anal', 'fcst', 'post', 'vrfy', 'arch'] - - if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: - tasks += ['postsnd'] - if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: - tasks += ['gempak'] - if cdump in ['gfs'] and do_awips in ['Y', 'YES']: - tasks += ['awips'] - - dict_resources = OrderedDict() - - for task in tasks: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump) - taskstr = '%s_%s' % (task.upper(), cdump.upper()) - - strings = [] - strings.append('\t\n' % (taskstr, queuestr)) - strings.append('\t\n' % (taskstr, wtimestr)) - strings.append('\t\n' % (taskstr, resstr)) - strings.append('\t\n' % (taskstr, memstr)) - strings.append('\t\n' % (taskstr, natstr)) - - dict_resources['%s%s' % (cdump, task)] = ''.join(strings) - - return dict_resources - - -def get_hyb_resources(dict_configs): - ''' - Create hybrid resource entities - ''' - - base = dict_configs['base'] - machine = base.get('machine', 'WCOSS_C') - lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() - eupd_cyc= base.get('EUPD_CYC', 'gdas').upper() - - dict_resources = OrderedDict() - - # These tasks can be run in either or both cycles - tasks1 = ['eobs', 'eomg', 'eupd'] - if lobsdiag_forenkf in ['.T.', '.TRUE.']: - tasks.remove('eomg') - - if eupd_cyc in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif eupd_cyc in ['GFS']: - cdumps = ['gfs'] - elif eupd_cyc in ['GDAS']: - cdumps = ['gdas'] - - for cdump in cdumps: - for task in tasks1: - - cfg = dict_configs['eobs'] if task in ['eomg'] else dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump) - - taskstr = '%s_%s' % (task.upper(), cdump.upper()) - - strings = [] - - strings.append('\t\n' % (taskstr, queuestr)) - strings.append('\t\n' % (taskstr, wtimestr)) - strings.append('\t\n' % (taskstr, resstr)) - strings.append('\t\n' % (taskstr, memstr)) - strings.append('\t\n' % (taskstr, natstr)) - - dict_resources['%s%s' % (cdump, task)] = ''.join(strings) - - - # These tasks are always run as part of the GDAS cycle - cdump = 'gdas' - tasks2 = ['ecen', 'efcs', 'epos', 'earc'] - for task in tasks2: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump) - - taskstr = '%s_%s' % (task.upper(), cdump.upper()) - - strings = [] - strings.append('\t\n' % (taskstr, queuestr)) - strings.append('\t\n' % (taskstr, wtimestr)) - strings.append('\t\n' % (taskstr, resstr)) - strings.append('\t\n' % (taskstr, memstr)) - strings.append('\t\n' % (taskstr, natstr)) - - dict_resources['%s%s' % (cdump, task)] = ''.join(strings) - - return dict_resources - - -def get_gdasgfs_tasks(dict_configs, cdump='gdas'): - ''' - Create GDAS or GFS tasks - ''' - - envars = [] - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump)) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - base = dict_configs['base'] - gfs_cyc = base.get('gfs_cyc', 0) - dohybvar = base.get('DOHYBVAR', 'NO').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper() - do_gempak = base.get('DO_GEMPAK', 'NO').upper() - do_awips = base.get('DO_AWIPS', 'NO').upper() - - dict_tasks = OrderedDict() - - # prep - deps = [] - dep_dict = {'type': 'metatask', 'name': '%spost' % 'gdas', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ROTDIR;/gdas.@Y@m@d/@H/gdas.t@Hz.atmf009.nemsio' - dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&DMPDIR;/@Y@m@d@H/%s/%s.t@Hz.updated.status.tm00.bufr_d' % (cdump, cdump) - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - gfs_enkf = True if eupd_cyc in ['BOTH', 'GFS'] and dohybvar in ['Y', 'YES'] else False - - if gfs_enkf and cdump in ['gfs']: - if gfs_cyc == 4: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies, cycledef='gdas') - - else: - task = wfu.create_wf_task('prep', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%sprep' % cdump] = task - - # anal - deps = [] - dep_dict = {'type': 'task', 'name': '%sprep' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - if dohybvar in ['y', 'Y', 'yes', 'YES']: - dep_dict = {'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - else: - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('anal', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%sanal' % cdump] = task - - # fcst - deps = [] - dep_dict = {'type': 'task', 'name': '%sanal' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - if cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - elif cdump in ['gfs']: - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%sfcst' % cdump] = task - - # post - deps = [] - data = '&ROTDIR;/%s.@Y@m@d/@H/%s.t@Hz.log#dep#.nemsio' % (cdump, cdump) - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks['%spost' % cdump] = task - - # vrfy - deps = [] - dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%svrfy' % cdump] = task - - - if cdump in ['gfs'] and do_bufrsnd in ['Y', 'YES']: - #postsnd - deps = [] - dep_dict = {'type': 'task', 'name': '%sfcst' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%spostsnd' % cdump] = task - - if cdump in ['gfs'] and do_awips in ['Y', 'YES']: - # awips - deps = [] - data = '&ROTDIR;/%s.@Y@m@d/@H/%s.t@Hz.sfluxgrb#dep#.grib2.idx' % (cdump, cdump) - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies, - metatask='awips', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks['%sawips' % cdump] = task - - if cdump in ['gfs'] and do_gempak in ['Y', 'YES']: - # gempak - deps = [] - dep_dict = {'type': 'metatask', 'name': '%spost' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%sgempak' % cdump] = task - - # arch - deps = [] - dep_dict = {'type': 'task', 'name': '%svrfy' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies) - - dict_tasks['%sarch' % cdump] = task - - return dict_tasks - - -def get_hyb_tasks(dict_configs, cycledef='enkf'): - ''' - Create Hybrid tasks - ''' - - # Determine groups based on ensemble size and grouping - base = dict_configs['base'] - nens = base['NMEM_ENKF'] - lobsdiag_forenkf = base.get('lobsdiag_forenkf', '.false.').upper() - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - - eobs = dict_configs['eobs'] - nens_eomg = eobs['NMEM_EOMGGRP'] - neomg_grps = nens / nens_eomg - EOMGGROUPS = ' '.join(['%02d' % x for x in range(1, neomg_grps + 1)]) - - efcs = dict_configs['efcs'] - nens_efcs = efcs['NMEM_EFCSGRP'] - nefcs_grps = nens / nens_efcs - EFCSGROUPS = ' '.join(['%02d' % x for x in range(1, nefcs_grps + 1)]) - - earc = dict_configs['earc'] - nens_earc = earc['NMEM_EARCGRP'] - nearc_grps = nens / nens_earc - EARCGROUPS = ' '.join(['%02d' % x for x in range(0, nearc_grps + 1)]) - - envars = [] - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - #envars.append(rocoto.create_envar(name='CDUMP', value='%s' % cdump)) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - ensgrp = rocoto.create_envar(name='ENSGRP', value='#grp#') - - dict_tasks = OrderedDict() - - if eupd_cyc in ['BOTH']: - cdumps = ['gfs', 'gdas'] - elif eupd_cyc in ['GFS']: - cdumps = ['gfs'] - elif eupd_cyc in ['GDAS']: - cdumps = ['gdas'] - - for cdump in cdumps: - - envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump) - envars1 = envars + [envar_cdump] - - # eobs - deps = [] - dep_dict = {'type': 'task', 'name': '%sprep' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': '%sepmn' % 'gdas', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('eobs', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks['%seobs' % cdump] = task - - # eomn, eomg - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - deps = [] - dep_dict = {'type': 'task', 'name': '%seobs' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - eomgenvars= envars1 + [ensgrp] - task = wfu.create_wf_task('eomg', cdump=cdump, envar=eomgenvars, dependency=dependencies, - metatask='eomn', varname='grp', varval=EOMGGROUPS, cycledef=cycledef) - - dict_tasks['%seomn' % cdump] = task - - # eupd - deps = [] - if lobsdiag_forenkf in ['.F.', '.FALSE.']: - dep_dict = {'type': 'metatask', 'name': '%seomn' % cdump} - else: - dep_dict = {'type': 'task', 'name': '%seobs' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('eupd', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks['%seupd' % cdump] = task - - # All hybrid tasks beyond this point are always executed in the GDAS cycle - cdump = 'gdas' - envar_cdump = rocoto.create_envar(name='CDUMP', value='%s' % cdump) - envars1 = envars + [envar_cdump] - cdump_eupd = 'gfs' if eupd_cyc in ['GFS'] else 'gdas' - - # ecen - deps = [] - dep_dict = {'type': 'task', 'name': '%sanal' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': '%seupd' % cdump_eupd} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('ecen', cdump=cdump, envar=envars1, dependency=dependencies, cycledef=cycledef) - - dict_tasks['%secen' % cdump] = task - - # efmn, efcs - deps = [] - dep_dict = {'type': 'task', 'name': '%secen' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) - efcsenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('efcs', cdump=cdump, envar=efcsenvars, dependency=dependencies, - metatask='efmn', varname='grp', varval=EFCSGROUPS, cycledef=cycledef) - - dict_tasks['%sefmn' % cdump] = task - - # epmn, epos - deps = [] - dep_dict = {'type': 'metatask', 'name': '%sefmn' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - eposenvars = envars1 + [fhrgrp] + [fhrlst] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_eposgroups(dict_configs['epos'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('epos', cdump=cdump, envar=eposenvars, dependency=dependencies, - metatask='epmn', varname=varname1, varval=varval1, vardict=vardict) - - dict_tasks['%sepmn' % cdump] = task - - # eamn, earc - deps = [] - dep_dict = {'type': 'metatask', 'name': '%sepmn' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - earcenvars = envars1 + [ensgrp] - task = wfu.create_wf_task('earc', cdump=cdump, envar=earcenvars, dependency=dependencies, - metatask='eamn', varname='grp', varval=EARCGROUPS, cycledef=cycledef) - - dict_tasks['%seamn' % cdump] = task - - return dict_tasks - - -def get_workflow_header(base): - ''' - Create the workflow header block - ''' - - strings = [] - - strings.append('\n') - strings.append(']>\n') - strings.append('\n') - strings.append('\n') - strings.append('\n') - strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') - strings.append('\n') - strings.append('\t\n') - strings.append('\t&SDATE; &SDATE; 06:00:00\n') - strings.append('\t&SDATE; &EDATE; 06:00:00\n') - strings.append('\t&SDATE; &EDATE; 06:00:00\n') - if base['gfs_cyc'] != 0: - strings.append('\t&SDATE_GFS; &EDATE_GFS; &INTERVAL_GFS;\n') - - strings.append('\n') - - return ''.join(strings) - - -def get_workflow_footer(): - ''' - Generate workflow footer - ''' - - strings = [] - strings.append('\n\n') - - return ''.join(strings) - - -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'],post['FHMAX_GFS_06'],post['FHMAX_GFS_12'],post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = ['f%03d' % f for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['%03d' % x for x in range(0, ngrps+1)]) - fhrdep = ' '.join(['f000'] + [f[-1] for f in fhrs]) - fhrlst = ' '.join(['anl'] + ['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_awipsgroups(awips, cdump='gdas'): - - fhmin = awips['FHMIN'] - fhmax = awips['FHMAX'] - fhout = awips['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([awips['FHMAX_GFS_00'],awips['FHMAX_GFS_06'],awips['FHMAX_GFS_12'],awips['FHMAX_GFS_18']]) - fhout = awips['FHOUT_GFS'] - fhmax_hf = awips['FHMAX_HF_GFS'] - fhout_hf = awips['FHOUT_HF_GFS'] - if fhmax > 240: - fhmax = 240 - if fhmax_hf > 240: - fhmax_hf = 240 - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) - - nawipsgrp = awips['NAWIPSGRP'] - ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) - - fhrs = ['f%03d' % f for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['%03d' % x for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - -def get_eposgroups(epos, cdump='gdas'): - - fhmin = epos['FHMIN_ENKF'] - fhmax = epos['FHMAX_ENKF'] - fhout = epos['FHOUT_ENKF'] - fhrs = range(fhmin, fhmax+fhout, fhout) - - neposgrp = epos['NEPOSGRP'] - ngrps = neposgrp if len(fhrs) > neposgrp else len(fhrs) - - fhrs = ['f%03d' % f for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['%03d' % x for x in range(0, ngrps)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - -def dict_to_strings(dict_in): - - strings = [] - for key in dict_in.keys(): - strings.append(dict_in[key]) - strings.append('\n') - - return ''.join(strings) - - -def create_xml(dict_configs): - ''' - Given an dictionary of sourced config files, - create the workflow XML - ''' - - base = dict_configs['base'] - dohybvar = base.get('DOHYBVAR', 'NO').upper() - gfs_cyc = base.get('gfs_cyc', 0) - eupd_cyc = base.get('EUPD_CYC', 'gdas').upper() - - # Start collecting workflow pieces - preamble = get_preamble() - definitions = get_definitions(base) - workflow_header = get_workflow_header(base) - workflow_footer = get_workflow_footer() - - # Get GDAS related entities, resources, workflow - dict_gdas_resources = get_gdasgfs_resources(dict_configs) - dict_gdas_tasks = get_gdasgfs_tasks(dict_configs) - - # Get hybrid related entities, resources, workflow - if dohybvar in ['Y', 'YES']: - dict_hyb_resources = get_hyb_resources(dict_configs) - dict_hyb_tasks = get_hyb_tasks(dict_configs) - - # Get GFS cycle related entities, resources, workflow - dict_gfs_resources = get_gdasgfs_resources(dict_configs, cdump='gfs') - dict_gfs_tasks = get_gdasgfs_tasks(dict_configs, cdump='gfs') - - # Put together the XML file - xmlfile = [] - - xmlfile.append(preamble) - - xmlfile.append(definitions) - - xmlfile.append(dict_to_strings(dict_gdas_resources)) - - if dohybvar in ['Y', 'YES']: - xmlfile.append(dict_to_strings(dict_hyb_resources)) - - if gfs_cyc != 0: - xmlfile.append(dict_to_strings(dict_gfs_resources)) - elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_resources['gfsprep']) - - xmlfile.append(workflow_header) - - xmlfile.append(dict_to_strings(dict_gdas_tasks)) - - if dohybvar in ['Y', 'YES']: - xmlfile.append(dict_to_strings(dict_hyb_tasks)) - - if gfs_cyc != 0: - xmlfile.append(dict_to_strings(dict_gfs_tasks)) - elif gfs_cyc == 0 and dohybvar in ['Y', 'YES'] and eupd_cyc in ['BOTH', 'GFS']: - xmlfile.append(dict_gfs_tasks['gfsprep']) - xmlfile.append('\n') - - xmlfile.append(wfu.create_firstcyc_task()) - - xmlfile.append(workflow_footer) - - # Write the XML file - fh = open('%s/%s.xml' % (base['EXPDIR'], base['PSLOT']), 'w') - fh.write(''.join(xmlfile)) - fh.close() - - return - - -if __name__ == '__main__': - main() - sys.exit(0) diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py deleted file mode 100755 index c5300e0f06..0000000000 --- a/ush/rocoto/setup_workflow_fcstonly.py +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### -''' - PROGRAM: - Create the ROCOTO workflow for a forecast only experiment given the configuration of the GFS parallel - - AUTHOR: - Rahul.Mahajan - rahul.mahajan@noaa.gov - - FILE DEPENDENCIES: - 1. config files for the parallel; e.g. config.base, config.fcst[.gfs], etc. - Without this dependency, the script will fail - - OUTPUT: - 1. PSLOT.xml: XML workflow - 2. PSLOT.crontab: crontab for ROCOTO run command - -''' - -import os -import sys -import numpy as np -from datetime import datetime -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import rocoto -import workflow_utils as wfu - - -taskplan = ['getic', 'fv3ic', 'fcst', 'post', 'vrfy', 'arch'] - -def main(): - parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('--expdir',help='full path to experiment directory containing config files', type=str, required=False, default=os.environ['PWD']) - parser.add_argument('--cdump',help='cycle to run forecasts', type=str, choices=['gdas', 'gfs'], default='gfs', required=False) - - args = parser.parse_args() - - configs = wfu.get_configs(args.expdir) - - _base = wfu.config_parser([wfu.find_config('config.base', configs)]) - - if not os.path.samefile(args.expdir,_base['EXPDIR']): - print 'MISMATCH in experiment directories!' - print 'config.base: EXPDIR = %s' % repr(_base['EXPDIR']) - print 'input arg: --expdir = %s' % repr(args.expdir) - sys.exit(1) - - dict_configs = wfu.source_configs(configs, taskplan) - - dict_configs['base']['CDUMP'] = args.cdump - - # First create workflow XML - create_xml(dict_configs) - - # Next create the crontab - wfu.create_crontab(dict_configs['base']) - - return - - -def get_preamble(): - ''' - Generate preamble for XML - ''' - - strings = [] - - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def get_definitions(base): - ''' - Create entities related to the experiment - ''' - - strings = [] - - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['PSLOT']) - strings.append('\t\n' % base['CDUMP']) - strings.append('\t\n' % base['CASE']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['SDATE'].strftime('%Y%m%d%H%M')) - strings.append('\t\n' % base['EDATE'].strftime('%Y%m%d%H%M')) - if base['INTERVAL'] is None: - print 'cycle INTERVAL cannot be None' - sys.exit(1) - strings.append('\t\n' % base['INTERVAL']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['RUN_ENVIR']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['EXPDIR']) - strings.append('\t\n' % base['ROTDIR']) - strings.append('\t\n' % base['ICSDIR']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['HOMEgfs']) - strings.append('\t\n' % base['BASE_JOB']) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n' % base['ACCOUNT']) - strings.append('\t\n' % base['QUEUE']) - strings.append('\t\n' % base['QUEUE_ARCH']) - strings.append('\t\n' % wfu.get_scheduler(base['machine'])) - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\t\n') - strings.append('\n') - - return ''.join(strings) - - -def get_resources(dict_configs, cdump='gdas'): - ''' - Create resource entities - ''' - - strings = [] - - strings.append('\t\n') - strings.append('\n') - - machine = dict_configs['base']['machine'] - - for task in taskplan: - - cfg = dict_configs[task] - - wtimestr, resstr, queuestr, memstr, natstr = wfu.get_resources(machine, cfg, task, cdump=cdump) - - taskstr = '%s_%s' % (task.upper(), cdump.upper()) - - strings.append('\t\n' % (taskstr, queuestr)) - strings.append('\t\n' % (taskstr, wtimestr)) - strings.append('\t\n' % (taskstr, resstr)) - strings.append('\t\n' % (taskstr, memstr)) - strings.append('\t\n' % (taskstr, natstr)) - - strings.append('\n') - - strings.append('\t\n') - - return ''.join(strings) - - -def get_postgroups(post, cdump='gdas'): - - fhmin = post['FHMIN'] - fhmax = post['FHMAX'] - fhout = post['FHOUT'] - - # Get a list of all forecast hours - if cdump in ['gdas']: - fhrs = range(fhmin, fhmax+fhout, fhout) - elif cdump in ['gfs']: - fhmax = np.max([post['FHMAX_GFS_00'],post['FHMAX_GFS_06'],post['FHMAX_GFS_12'],post['FHMAX_GFS_18']]) - fhout = post['FHOUT_GFS'] - fhmax_hf = post['FHMAX_HF_GFS'] - fhout_hf = post['FHOUT_HF_GFS'] - fhrs_hf = range(fhmin, fhmax_hf+fhout_hf, fhout_hf) - fhrs = fhrs_hf + range(fhrs_hf[-1]+fhout, fhmax+fhout, fhout) - - npostgrp = post['NPOSTGRP'] - ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) - - fhrs = ['f%03d' % f for f in fhrs] - fhrs = np.array_split(fhrs, ngrps) - fhrs = [f.tolist() for f in fhrs] - - fhrgrp = ' '.join(['%03d' % x for x in range(1, ngrps+1)]) - fhrdep = ' '.join([f[-1] for f in fhrs]) - fhrlst = ' '.join(['_'.join(f) for f in fhrs]) - - return fhrgrp, fhrdep, fhrlst - - -def get_workflow(dict_configs, cdump='gdas'): - ''' - Create tasks for forecast only workflow - ''' - - envars = [] - envars.append(rocoto.create_envar(name='RUN_ENVIR', value='&RUN_ENVIR;')) - envars.append(rocoto.create_envar(name='HOMEgfs', value='&HOMEgfs;')) - envars.append(rocoto.create_envar(name='EXPDIR', value='&EXPDIR;')) - envars.append(rocoto.create_envar(name='CDATE', value='@Y@m@d@H')) - envars.append(rocoto.create_envar(name='CDUMP', value='&CDUMP;')) - envars.append(rocoto.create_envar(name='PDY', value='@Y@m@d')) - envars.append(rocoto.create_envar(name='cyc', value='@H')) - - tasks = [] - - # getics - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/pgbanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/siganl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/sfcanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies = rocoto.create_dependency(dep_condition='not', dep=deps) - task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # chgres - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/siganl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/sfcanl.&CDUMP;.@Y@m@d@H' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - deps = rocoto.create_dependency(dep_condition='and', dep=deps) - dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps) - - deps = [] - deps.append(dependencies) - deps.append(dependencies2) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task = wfu.create_wf_task('fv3ic', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # fcst - deps = [] - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc' - dep_dict = {'type':'data', 'data':data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('fcst', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # post - deps = [] - data = '&ROTDIR;/%s.@Y@m@d/@H/%s.t@Hz.log#dep#.nemsio' % (cdump, cdump) - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#') - fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#') - ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;') - postenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR] - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = get_postgroups(dict_configs['post'], cdump=cdump) - vardict = {varname2: varval2, varname3: varval3} - task = wfu.create_wf_task('post', cdump=cdump, envar=postenvars, dependency=dependencies, - metatask='post', varname=varname1, varval=varval1, vardict=vardict) - tasks.append(task) - tasks.append('\n') - - # vrfy - deps = [] - dep_dict = {'type':'metatask', 'name':'%spost' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - task = wfu.create_wf_task('vrfy', cdump=cdump, envar=envars, dependency=dependencies) - tasks.append(task) - tasks.append('\n') - - # arch - deps = [] - dep_dict = {'type':'task', 'name':'%svrfy' % cdump} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True) - tasks.append(task) - tasks.append('\n') - - return ''.join(tasks) - - -def get_workflow_body(dict_configs, cdump='gdas'): - ''' - Create the workflow body - ''' - - strings = [] - - strings.append('\n') - strings.append(']>\n') - strings.append('\n') - strings.append('\n') - strings.append('\n') - strings.append('\t&EXPDIR;/logs/@Y@m@d@H.log\n') - strings.append('\n') - strings.append('\t\n') - strings.append('\t&SDATE; &EDATE; &INTERVAL;\n' % cdump) - strings.append('\n') - strings.append(get_workflow(dict_configs, cdump=cdump)) - strings.append('\n') - strings.append('\n') - - return ''.join(strings) - - -def create_xml(dict_configs): - ''' - Given an experiment directory containing config files and - XML directory containing XML templates, create the workflow XML - ''' - - - dict_configs['base']['INTERVAL'] = wfu.get_gfs_interval(dict_configs['base']['gfs_cyc']) - base = dict_configs['base'] - - preamble = get_preamble() - definitions = get_definitions(base) - resources = get_resources(dict_configs, cdump=base['CDUMP']) - workflow = get_workflow_body(dict_configs, cdump=base['CDUMP']) - - # Start writing the XML file - fh = open('%s/%s.xml' % (base['EXPDIR'], base['PSLOT']), 'w') - - fh.write(preamble) - fh.write(definitions) - fh.write(resources) - fh.write(workflow) - - fh.close() - - return - -if __name__ == '__main__': - main() - sys.exit(0) diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py deleted file mode 100755 index 6bbaaf921e..0000000000 --- a/ush/rocoto/workflow_utils.py +++ /dev/null @@ -1,341 +0,0 @@ -#!/usr/bin/env python - -############################################################### -# < next few lines under version control, D O N O T E D I T > -# $Date$ -# $Revision$ -# $Author$ -# $Id$ -############################################################### -''' - Module containing functions all workflow setups require -''' -import random -import os -import sys -import glob -import subprocess -import numpy as np -from distutils.spawn import find_executable -from datetime import datetime, timedelta -import rocoto - -DATE_ENV_VARS=['CDATE','SDATE','EDATE'] -SCHEDULER_MAP={'ZEUS':'moabtorque', - 'THEIA':'moabtorque', - 'WCOSS':'lsf', - 'WCOSS_DELL_P3':'lsf', - 'WCOSS_C':'lsfcray'} - -class UnknownMachineError(Exception): pass -class UnknownConfigError(Exception): pass -class ShellScriptException(Exception): - def __init__(self,scripts,errors): - self.scripts = scripts - self.errors = errors - super(ShellScriptException,self).__init__( - str(errors)+ - ': error processing'+ - (' '.join(scripts))) - -def get_shell_env(scripts): - vars=dict() - runme=''.join([ 'source %s ; '%(s,) for s in scripts ]) - magic='--- ENVIRONMENT BEGIN %d ---'%random.randint(0,64**5) - runme+='/bin/echo -n "%s" ; /usr/bin/env -0'%(magic,) - with open('/dev/null','wb+') as null: - env=subprocess.Popen(runme,shell=True,stdin=null.fileno(), - stdout=subprocess.PIPE) - (out,err)=env.communicate() - begin=out.find(magic) - if begin<0: - raise ShellScriptException(scripts,'Cannot find magic string; ' - 'at least one script failed: '+repr(out)) - for entry in out[begin+len(magic):].split('\x00'): - iequal=entry.find('=') - vars[entry[0:iequal]] = entry[iequal+1:] - return vars - -def get_script_env(scripts): - default_env=get_shell_env([]) - and_script_env=get_shell_env(scripts) - vars_just_in_script=set(and_script_env)-set(default_env) - union_env=dict(default_env) - union_env.update(and_script_env) - return dict([ (v,union_env[v]) for v in vars_just_in_script ]) - -def cast_or_not(type,value): - try: - return type(value) - except ValueError: - return value - -def get_configs(expdir): - """ - Given an experiment directory containing config files, - return a list of configs minus the ones ending with ".default" - """ - result=list() - for config in glob.glob('%s/config.*' % expdir): - if not config.endswith('.default'): - result.append(config) - return result - -def find_config(config_name, configs): - - for config in configs: - if config_name == os.path.basename(config): - return config - - raise UnknownConfigError("%s does not exist (known: %s), ABORT!" % ( - config_name,repr(config_name))) - -def source_configs(configs, tasks): - ''' - Given list of config files, source them - and return a dictionary for each task - Every task depends on config.base - ''' - - dict_configs = {} - - # Return config.base as well - dict_configs['base'] = config_parser([find_config('config.base', configs)]) - - # Source the list of input tasks - for task in tasks: - - files = [] - - files.append(find_config('config.base', configs)) - - if task in ['eobs', 'eomg']: - files.append(find_config('config.anal', configs)) - files.append(find_config('config.eobs', configs)) - elif task in ['eupd']: - files.append(find_config('config.anal', configs)) - files.append(find_config('config.eupd', configs)) - elif task in ['efcs']: - files.append(find_config('config.fcst', configs)) - files.append(find_config('config.efcs', configs)) - else: - files.append(find_config('config.%s' % task, configs)) - - print 'sourcing config.%s' % task - dict_configs[task] = config_parser(files) - - return dict_configs - - -def config_parser(files): - """ - Given the name of config file, key-value pair of all variables in the config file is returned as a dictionary - :param files: config file or list of config files - :type files: list or str or unicode - :return: Key value pairs representing the environment variables defined - in the script. - :rtype: dict - """ - if isinstance(files,basestring): - files=[files] - varbles=dict() - for key,value in get_script_env(files).iteritems(): - if key in DATE_ENV_VARS: # likely a date, convert to datetime - varbles[key] = datetime.strptime(value,'%Y%m%d%H') - elif '.' in value: # Likely a number and that too a float - varbles[key] = cast_or_not(float,value) - else: # Still could be a number, may be an integer - varbles[key] = cast_or_not(int,value) - - return varbles - -def get_scheduler(machine): - """Determine the scheduler""" - try: - return SCHEDULER_MAP[machine] - except KeyError: - raise UnknownMachineError('Unknown machine: %s'%(machine,)) - - -def create_wf_task(task, cdump='gdas', cycledef=None, envar=None, dependency=None, \ - metatask=None, varname=None, varval=None, vardict=None, \ - final=False): - - if metatask is None: - taskstr = '%s' % task - else: - taskstr = '%s#%s#' % (task, varname) - metataskstr = '%s%s' % (cdump, metatask) - metatask_dict = {'metataskname': metataskstr, \ - 'varname': '%s' % varname, \ - 'varval': '%s' % varval, \ - 'vardict': vardict} - - taskstr = '%s%s' % (cdump, taskstr) - cycledefstr = cdump if cycledef is None else cycledef - - task_dict = {'taskname': '%s' % taskstr, \ - 'cycledef': '%s' % cycledefstr, \ - 'maxtries': '&MAXTRIES;', \ - 'command': '&JOBS_DIR;/%s.sh' % task, \ - 'jobname': '&PSLOT;_%s_@H' % taskstr, \ - 'account': '&ACCOUNT;', \ - 'queue': '&QUEUE_%s_%s;' % (task.upper(), cdump.upper()), \ - 'walltime': '&WALLTIME_%s_%s;' % (task.upper(), cdump.upper()), \ - 'native': '&NATIVE_%s_%s;' % (task.upper(), cdump.upper()), \ - 'memory': '&MEMORY_%s_%s;' % (task.upper(), cdump.upper()), \ - 'resources': '&RESOURCES_%s_%s;' % (task.upper(), cdump.upper()), \ - 'log': '&ROTDIR;/logs/@Y@m@d@H/%s.log' % taskstr, \ - 'envar': envar, \ - 'dependency': dependency, \ - 'final': final} - - if metatask is None: - task = rocoto.create_task(task_dict) - else: - task = rocoto.create_metatask(task_dict, metatask_dict) - task = ''.join(task) - - return task - - -def create_firstcyc_task(cdump='gdas'): - ''' - This task is needed to run to finalize the first half cycle - ''' - - task = 'firstcyc' - taskstr = '%s' % task - - deps = [] - data = '&EXPDIR;/logs/@Y@m@d@H.log' - dep_dict = {'type':'data', 'data':data, 'offset':'24:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type':'cycleexist', 'condition':'not', 'offset':'-06:00:00'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - - task_dict = {'taskname': '%s' % taskstr, \ - 'cycledef': 'first', \ - 'maxtries': '&MAXTRIES;', \ - 'final' : True, \ - 'command': 'sleep 1', \ - 'jobname': '&PSLOT;_%s_@H' % taskstr, \ - 'account': '&ACCOUNT;', \ - 'queue': '&QUEUE_ARCH;', \ - 'walltime': '&WALLTIME_ARCH_%s;' % cdump.upper(), \ - 'native': '&NATIVE_ARCH_%s;' % cdump.upper(), \ - 'resources': '&RESOURCES_ARCH_%s;' % cdump.upper(), \ - 'log': '&ROTDIR;/logs/@Y@m@d@H/%s.log' % taskstr, \ - 'dependency': dependencies} - - task = rocoto.create_task(task_dict) - - return ''.join(task) - - -def get_gfs_interval(gfs_cyc): - ''' - return interval in hours based on gfs_cyc - ''' - - # Get interval from cyc_input - if gfs_cyc == 0: - interval = None - if gfs_cyc == 1: - interval = '24:00:00' - elif gfs_cyc == 2: - interval = '12:00:00' - elif gfs_cyc == 4: - interval = '06:00:00' - - return interval - - -def get_resources(machine, cfg, task, cdump='gdas'): - - if cdump in ['gfs'] and 'wtime_%s_gfs' % task in cfg.keys(): - wtimestr = cfg['wtime_%s_gfs' % task] - else: - wtimestr = cfg['wtime_%s' % task] - - ltask = 'eobs' if task in ['eomg'] else task - - memory = cfg.get('memory_%s' % ltask, None) - - if cdump in ['gfs'] and 'npe_%s_gfs' % task in cfg.keys(): - tasks = cfg['npe_%s_gfs' % ltask] - else: - tasks = cfg['npe_%s' % ltask] - - if cdump in ['gfs'] and 'npe_node_%s_gfs' % task in cfg.keys(): - ppn = cfg['npe_node_%s_gfs' % ltask] - else: - ppn = cfg['npe_node_%s' % ltask] - - if machine in [ 'WCOSS_DELL_P3']: - threads = cfg['nth_%s' % ltask] - - nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn))) - - memstr = '' if memory is None else str(memory) - natstr = '' - - if machine in ['ZEUS', 'THEIA', 'WCOSS_C', 'WCOSS_DELL_P3']: - resstr = '%d:ppn=%d' % (nodes, ppn) - - if machine in ['WCOSS_C'] and task in ['arch', 'earc', 'getic']: - resstr += '' - - if machine in ['WCOSS_DELL_P3']: - natstr = "-R 'affinity[core(%d)]'" % (threads) - - if task in ['arch', 'earc', 'getic']: - natstr = "-R 'affinity[core(1)]'" - - elif machine in ['WCOSS']: - resstr = '%d' % tasks - - queuestr = '&QUEUE_ARCH;' if task in ['arch', 'earc', 'getic'] else '&QUEUE;' - - return wtimestr, resstr, queuestr, memstr, natstr - - -def create_crontab(base, cronint=5): - ''' - Create crontab to execute rocotorun every cronint (5) minutes - ''' - - # No point creating a crontab if rocotorun is not available. - rocotoruncmd = find_executable('rocotorun') - if rocotoruncmd is None: - print 'Failed to find rocotorun, crontab will not be created' - return - - cronintstr = '*/%d * * * *' % cronint - rocotorunstr = '%s -d %s/%s.db -w %s/%s.xml' % (rocotoruncmd, base['EXPDIR'], base['PSLOT'], base['EXPDIR'], base['PSLOT']) - - # On WCOSS, rocoto module needs to be loaded everytime cron runs - if base['machine'] in ['WCOSS']: - rocotoloadstr = '. /usrx/local/Modules/default/init/sh; module use -a /usrx/local/emc_rocoto/modulefiles; module load rocoto/20170119-master)' - rocotorunstr = '(%s %s)' % (rocotoloadstr, rocotorunstr) - - try: - REPLYTO = os.environ['REPLYTO'] - except: - REPLYTO = '' - - strings = [] - - strings.append('\n') - strings.append('#################### %s ####################\n' % base['PSLOT']) - strings.append('MAILTO="%s"\n' % REPLYTO) - strings.append('%s %s\n' % (cronintstr, rocotorunstr)) - strings.append('#################################################################\n') - strings.append('\n') - - fh = open(os.path.join(base['EXPDIR'], '%s.crontab' % base['PSLOT']), 'w') - fh.write(''.join(strings)) - fh.close() - - return From 74defd49e92114a2ab0aeec083e94e26d9743c5f Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 1 Apr 2022 14:17:58 -0400 Subject: [PATCH 143/203] Fix bug for nco we2e tests on Orion; re-organize we2e input data and nco we2e tests (#713) * Update machine script for orion * Update machine script for wcoss_dell_p3 * Update we2e run script for wcoss and orion * Reorganize nco we2e tests * remove machine based logic * Add symlink for nco inline post test --- tests/WE2E/run_WE2E_tests.sh | 5 ++-- ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} | 5 ++-- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 26 ----------------- ..._25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 25 ---------------- ...CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 6 ++-- ...GFS_suite_GFS_v15_thompson_mynn_lam3km.sh} | 10 +++++-- ..._ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh | 26 ----------------- .../wflow_features/config.nco_ensemble.sh | 10 +++---- .../wflow_features/config.nco_inline_post.sh | 29 +------------------ ush/machine/orion.sh | 15 ++++++++-- ush/machine/wcoss_dell_p3.sh | 10 +++---- 11 files changed, 41 insertions(+), 126 deletions(-) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/{config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh => config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} (78%) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/{config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh => config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh} (63%) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh mode change 100644 => 120000 tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 4ee8b504ed..81757dc91d 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -941,7 +941,7 @@ machine (MACHINE): MACHINE= \"${MACHINE}\"" fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" fi if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ @@ -959,7 +959,8 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then + if [[ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" && + ( "$MACHINE" = "HERA" || "$MACHINE" = "WCOSS_DELL_P3" || "$MACHINE" = "ORION" ) ]]; then EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" fi # diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 78% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 141a9857ce..f9eea07295 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -3,7 +3,7 @@ # ------------------------ # # This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_13km grid using the GFS_v15p2 physics +# successfully on the RRFS_CONUS_13km grid using the GFS_v16 physics # suite with ICs and LBCs derived from the FV3GFS. # @@ -11,12 +11,13 @@ RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_13km" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" +CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" +USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh deleted file mode 100644 index 1b7d92162c..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 physics -# suite with ICs and LBCs derived from the FV3GFS. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" - -DATE_FIRST_CYCL="20190615" -DATE_LAST_CYCL="20190615" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh deleted file mode 100644 index 656a4d0da9..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ /dev/null @@ -1,25 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_25km grid using the FV3_GFS_v15p2 physics suite -# with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20200208" -DATE_LAST_CYCL="20200208" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 300dcd6664..0a2d426fa2 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -17,9 +17,11 @@ EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" -DATE_FIRST_CYCL="20200208" -DATE_LAST_CYCL="20200208" +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" + +WRITE_DOPOST="TRUE" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh similarity index 63% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh index 64c267636f..6a02d4f119 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh @@ -3,21 +3,25 @@ # ------------------------ # # This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_3km grid using the GFS_v16 physics -# suite with ICs and LBCs derived from the FV3GFS. +# successfully on the RRFS_CONUS_3km grid using the GFS_v15_thompson_mynn_lam3km +# physics suite with ICs and LBCs derived from the FV3GFS. # RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_GFS_v16" +CCPP_PHYS_SUITE="FV3_GFS_v15_thompson_mynn_lam3km" + +USE_MERRA_CLIMO="TRUE" EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" +USE_USER_STAGED_EXTRN_FILES="TRUE" + DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" CYCL_HRS=( "00" ) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh deleted file mode 100644 index aac14a4794..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ /dev/null @@ -1,26 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_3km grid using the GFS_v15p2 physics -# suite with ICs and LBCs derived from the FV3GFS. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_3km" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" - -DATE_FIRST_CYCL="20190615" -DATE_LAST_CYCL="20190615" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh index 3b9c93f986..0f43e9e1dd 100644 --- a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh +++ b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh @@ -17,16 +17,16 @@ RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" USE_USER_STAGED_EXTRN_FILES="TRUE" -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190902" -CYCL_HRS=( "12" "18" ) +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190702" +CYCL_HRS=( "00" "12" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh deleted file mode 100644 index 003b082340..0000000000 --- a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh +++ /dev/null @@ -1,28 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test checks the capability of the workflow to use the inline -# post option (WRITE_DOPOST) in model_configure for 'nco' mode. -# - -RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -WRITE_DOPOST="TRUE" - -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" - - diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh new file mode 120000 index 0000000000..35901eeb53 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh @@ -0,0 +1 @@ +../grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh \ No newline at end of file diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index 5f4abc0d9c..5c872410d4 100644 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -48,7 +48,7 @@ FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/work/noaa/fv3-cam/UFS_SRW_App/FIXLAM_NCO_BASE"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -56,5 +56,16 @@ RUN_CMD_UTILS="srun" RUN_CMD_FCST='srun -n ${PE_MEMBER01}' RUN_CMD_POST="srun" +# MET/METplus-Related Paths +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.0"} +METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.0.0"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} + # Test Data Locations -TEST_EXTRN_MDL_SOURCE_BASEDIR=/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files +TEST_PREGEN_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/FV3LAM_pregen" +TEST_COMINgfs="/work/noaa/fv3-cam/UFS_SRW_App/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/develop/model_data" + diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 03656438fb..811c3ae8b2 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -60,7 +60,7 @@ FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fi FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen"} +FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FIXLAM_NCO_BASE"} # Run commands for executables RUN_CMD_SERIAL="mpirun" @@ -71,12 +71,12 @@ RUN_CMD_POST="mpirun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} # Test Data Locations TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" TEST_COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" +TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/model_data" From 788667c49ab7ceb241960bbec088ec4a3ead6c09 Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Fri, 1 Apr 2022 12:46:15 -0600 Subject: [PATCH 144/203] Added stand-alone verification scripts (feature/issue_683_standaloneVX) (#726) * Grid-stat and point-stat run scripts. * Stand-alone scripts for verification. * Added comments to gridvx scripts. * Added qsub_job.sh and added comments to provide context on running Vx. --- ush/wrappers/qsub_job.sh | 19 ++++++++++++++++--- ush/wrappers/run_gridensvx.sh | 22 ++++++++++++++++++++++ ush/wrappers/run_gridvx.sh | 19 +++++++++++++++++++ ush/wrappers/run_pointensvx.sh | 20 ++++++++++++++++++++ ush/wrappers/run_pointvx.sh | 17 +++++++++++++++++ ush/wrappers/sq_job.sh | 21 ++++++++++++++++++--- 6 files changed, 112 insertions(+), 6 deletions(-) create mode 100755 ush/wrappers/run_gridensvx.sh create mode 100755 ush/wrappers/run_gridvx.sh create mode 100755 ush/wrappers/run_pointensvx.sh create mode 100755 ush/wrappers/run_pointvx.sh diff --git a/ush/wrappers/qsub_job.sh b/ush/wrappers/qsub_job.sh index 797590509c..15ec7eb140 100755 --- a/ush/wrappers/qsub_job.sh +++ b/ush/wrappers/qsub_job.sh @@ -1,11 +1,12 @@ #!/bin/sh -#PBS -A P48503002 +#PBS -A XXXXXXXXX #PBS -q regular #PBS -l select=1:mpiprocs=24:ncpus=24 +##PBS -l select=1:mpiprocs=1:ncpus=1 # USE FOR MET VERIFICATION #PBS -l walltime=02:30:00 #PBS -N run_make_grid -#PBS -j oe -o /glade/scratch/carson/ufs/expt_dirs/test_1/log/run_make_grid.log -cd /glade/scratch/carson/ufs/expt_dirs/test_1 +#PBS -j oe -o /path/to/exptdir/log/run_make_grid.log # NEED TO SET +cd /path/to/exptdir # NEED TO SET set -x # source /etc/profile.d/modules.sh @@ -31,3 +32,15 @@ module load esmf/8.0.0 ##module load esmf/8.0.0_bs50 # ./run_make_grid.sh +# +# +# Additional modules are needed for MET verification jobs +# +#module use /glade/p/ral/jntp/MET/MET_releases/modulefiles +#module load met/10.0.0 +#ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics + +#./run_pointvx.sh # Run grod-to-point deterministic vx +#./run_gridvx.sh # Run grid-stat deterministic vx +#./run_pointensvx.sh # Run grid-to-point ensemble/probabilsitic vx +#./rungridensvx.sh # Run grid-to-grid ensemble/probabilsitic vx diff --git a/ush/wrappers/run_gridensvx.sh b/ush/wrappers/run_gridensvx.sh new file mode 100755 index 0000000000..d6a3c76aff --- /dev/null +++ b/ush/wrappers/run_gridensvx.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# Stand-alone script to run grid-to-grid ensemble verification +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +set -x +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${CYCL_HRS} +export PDY=${DATE_FIRST_CYCL} +export OBS_DIR=${MRMS_OBS_DIR} # CCPA_OBS_DIR MRMS_OBS_DIR +export VAR="REFC" # APCP REFC RETOP +export ACCUM="" # 01 03 06 24 --> leave empty for REFC and RETOP + +export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID_MEAN + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID_PROB + diff --git a/ush/wrappers/run_gridvx.sh b/ush/wrappers/run_gridvx.sh new file mode 100755 index 0000000000..a96383f6ed --- /dev/null +++ b/ush/wrappers/run_gridvx.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# Stand-alone script to run grid-to-grid verification +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +set -x +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${CYCL_HRS} +export PDY=${DATE_FIRST_CYCL} +export SLASH_ENSMEM_SUBDIR="" # When running with do_ensemble = true, need to run for each member, e.g., "/mem1" +export OBS_DIR=${CCPA_OBS_DIR} # CCPA_OBS_DIR MRMS_OBS_DIR +export VAR="APCP" # APCP REFC RETOP +export ACCUM="06" # 01 03 06 24 --> leave empty for REFC and RETOP + +export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` + +${JOBSDIR}/JREGIONAL_RUN_VX_GRIDSTAT + diff --git a/ush/wrappers/run_pointensvx.sh b/ush/wrappers/run_pointensvx.sh new file mode 100755 index 0000000000..142d9ec833 --- /dev/null +++ b/ush/wrappers/run_pointensvx.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Stand-alone script to run grid-to-point ensemble verification +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +set -x +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${CYCL_HRS} +export PDY=${DATE_FIRST_CYCL} +export OBS_DIR=${NDAS_OBS_DIR} + +export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT_MEAN + +${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT_PROB + diff --git a/ush/wrappers/run_pointvx.sh b/ush/wrappers/run_pointvx.sh new file mode 100755 index 0000000000..527c16f37d --- /dev/null +++ b/ush/wrappers/run_pointvx.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Stand-alone script to run grid-to-point verification +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +set -x +source ${GLOBAL_VAR_DEFNS_FP} +export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} +export CYCLE_DIR=${EXPTDIR}/${CDATE} +export cyc=${CYCL_HRS} +export PDY=${DATE_FIRST_CYCL} +export SLASH_ENSMEM_SUBDIR="" # When running with do_ensemble = true, need to run for each member, e.g., "/mem1" +export OBS_DIR=${NDAS_OBS_DIR} + +export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` + +${JOBSDIR}/JREGIONAL_RUN_VX_POINTSTAT + diff --git a/ush/wrappers/sq_job.sh b/ush/wrappers/sq_job.sh index 394bfabe23..a9fa3c1805 100755 --- a/ush/wrappers/sq_job.sh +++ b/ush/wrappers/sq_job.sh @@ -1,11 +1,12 @@ #!/bin/sh -#SBATCH -e /scratch1/BMC/gmtb/Laurie.Carson/expt_dirs/test_2/log/run_make_grid.log -#SBATCH --account=gmtb +#SBATCH -e /path/to/exptdir/log/run_make_grid.log # NEED TO SET +#SBATCH --account=XXXXXXXXX #SBATCH --qos=batch #SBATCH --ntasks=48 +##SBATCH --ntasks=1 # USE FOR MET VERIFICATION #SBATCH --time=20 #SBATCH --job-name="run_make_grid" -cd /scratch1/BMC/gmtb/Laurie.Carson/expt_dirs/test_2 +cd /path/to/exptdir # NEED TO SET set -x . /apps/lmod/lmod/init/sh @@ -41,3 +42,17 @@ module load miniconda3 conda activate regional_workflow ./run_make_grid.sh +# +# +# Additional modules are needed for MET verification jobs +# +#module use -a /contrib/anaconda/modulefiles +#module load intel/18.0.5.274 +#module load anaconda/latest +#module use -a /contrib/met/modulefiles/ +#module load met/10.0.0 + +#./run_pointvx.sh # Run grod-to-point deterministic vx +#./run_gridvx.sh # Run grid-stat deterministic vx +#./run_pointensvx.sh # Run grid-to-point ensemble/probabilsitic vx +#./run_gridensvx.sh # Run grid-to-grid ensemble/probabilsitic vx From 832c93215d1d23ae329f04a730d456809b020a06 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 1 Apr 2022 22:55:58 -0400 Subject: [PATCH 145/203] remove machine base logic (#727) --- tests/WE2E/run_WE2E_tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 81757dc91d..c162a51004 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -959,8 +959,7 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" - if [[ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" && - ( "$MACHINE" = "HERA" || "$MACHINE" = "WCOSS_DELL_P3" || "$MACHINE" = "ORION" ) ]]; then + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" fi # From 251fd79d034566bebca7289a0af282e4ab87aecd Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 4 Apr 2022 12:49:16 -0400 Subject: [PATCH 146/203] Allow user-defined file names for input template files (#717) * Allow multiple template names * parameterize file_TMPL_FN and add a we2e test * Increase maxtries_task for make_grid/orog/sfc_climo * Modify file name and description --- .../config.specify_template_filenames.sh | 30 ++++++++++ ush/config_defaults.sh | 58 ++++++++++--------- ush/setup.sh | 27 +++++++-- 3 files changed, 84 insertions(+), 31 deletions(-) create mode 100644 tests/WE2E/test_configs/wflow_features/config.specify_template_filenames.sh diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_template_filenames.sh b/tests/WE2E/test_configs/wflow_features/config.specify_template_filenames.sh new file mode 100644 index 0000000000..81185425c6 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.specify_template_filenames.sh @@ -0,0 +1,30 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to use user-defined +# template files. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + +DATA_TABLE_TMPL_FN="data_table" +DIAG_TABLE_TMPL_FN="diag_table" +FIELD_TABLE_TMPL_FN="field_table" +MODEL_CONFIG_TMPL_FN="model_configure" +NEMS_CONFIG_TMPL_FN="nems.configure" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 466344f682..be96316d75 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -343,28 +343,34 @@ DOT_OR_USCORE="_" # from which the namelist files for each of the enesemble members are # generated. # -# DIAG_TABLE_FN: -# Name of file that specifies the fields that the forecast model will -# output. -# -# FIELD_TABLE_FN: -# Name of file that specifies the tracers that the forecast model will -# read in from the IC/LBC files. -# -# DATA_TABLE_FN: -# Name of file that specifies ??? -# -# MODEL_CONFIG_FN: -# Name of file that specifies ??? -# -# NEMS_CONFIG_FN: -# Name of file that specifies ??? -# # FV3_EXEC_FN: # Name to use for the forecast model executable when it is copied from # the directory in which it is created in the build step to the executables # directory (EXECDIR; this is set during experiment generation). # +# DIAG_TABLE_TMPL_FN: +# Name of a template file that specifies the output fields of the forecast +# model (ufs-weather-model: diag_table) followed by [dot_ccpp_phys_suite]. +# Its default value is the name of the file that the ufs weather model +# expects to read in. +# +# FIELD_TABLE_TMPL_FN: +# Name of a template file that specifies the tracers in IC/LBC files of the +# forecast model (ufs-weather-mode: field_table) followed by [dot_ccpp_phys_suite]. +# Its default value is the name of the file that the ufs weather model expects +# to read in. +# +# MODEL_CONFIG_TMPL_FN: +# Name of a template file that contains settings and configurations for the +# NUOPC/ESMF main component (ufs-weather-model: model_config). Its default +# value is the name of the file that the ufs weather model expects to read in. +# +# NEMS_CONFIG_TMPL_FN: +# Name of a template file that contains information about the various NEMS +# components and their run sequence (ufs-weather-model: nems.configure). +# Its default value is the name of the file that the ufs weather model expects +# to read in. +# # FCST_MODEL: # Name of forecast model (default=ufs-weather-model) # @@ -409,18 +415,18 @@ EXPT_CONFIG_FN="config.sh" RGNL_GRID_NML_FN="regional_grid.nml" -DATA_TABLE_FN="data_table" -DIAG_TABLE_FN="diag_table" -FIELD_TABLE_FN="field_table" FV3_NML_BASE_SUITE_FN="input.nml.FV3" FV3_NML_YAML_CONFIG_FN="FV3.input.yml" FV3_NML_BASE_ENS_FN="input.nml.base_ens" -MODEL_CONFIG_FN="model_configure" -NEMS_CONFIG_FN="nems.configure" FV3_EXEC_FN="ufs_model" -FCST_MODEL="ufs-weather-model" +DATA_TABLE_TMPL_FN="" +DIAG_TABLE_TMPL_FN="" +FIELD_TABLE_TMPL_FN="" +MODEL_CONFIG_TMPL_FN="" +NEMS_CONFIG_TMPL_FN="" +FCST_MODEL="ufs-weather-model" WFLOW_XML_FN="FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN="var_defns.sh" EXTRN_MDL_ICS_VAR_DEFNS_FN="extrn_mdl_ics_var_defns.sh" @@ -1646,9 +1652,9 @@ WTIME_VX_ENSPOINT_PROB="01:00:00" # # Maximum number of attempts. # -MAXTRIES_MAKE_GRID="1" -MAXTRIES_MAKE_OROG="1" -MAXTRIES_MAKE_SFC_CLIMO="1" +MAXTRIES_MAKE_GRID="2" +MAXTRIES_MAKE_OROG="2" +MAXTRIES_MAKE_SFC_CLIMO="2" MAXTRIES_GET_EXTRN_ICS="1" MAXTRIES_GET_EXTRN_LBCS="1" MAXTRIES_MAKE_ICS="1" diff --git a/ush/setup.sh b/ush/setup.sh index 01470963d5..cf76d1d2bd 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1216,11 +1216,22 @@ fi # dot_ccpp_phys_suite_or_null=".${CCPP_PHYS_SUITE}" -DATA_TABLE_TMPL_FN="${DATA_TABLE_FN}" -DIAG_TABLE_TMPL_FN="${DIAG_TABLE_FN}${dot_ccpp_phys_suite_or_null}" -FIELD_TABLE_TMPL_FN="${FIELD_TABLE_FN}${dot_ccpp_phys_suite_or_null}" -MODEL_CONFIG_TMPL_FN="${MODEL_CONFIG_FN}" -NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_FN}" +# Names of input files that the forecast model (ufs-weather-model) expects +# to read in. These should only be changed if the input file names in the +# forecast model code are changed. +#---------------------------------- +DATA_TABLE_FN="data_table" +DIAG_TABLE_FN="diag_table" +FIELD_TABLE_FN="field_table" +MODEL_CONFIG_FN="model_configure" +NEMS_CONFIG_FN="nems.configure" +#---------------------------------- + +DATA_TABLE_TMPL_FN="${DATA_TABLE_TMPL_FN:-${DATA_TABLE_FN}}" +DIAG_TABLE_TMPL_FN="${DIAG_TABLE_TMPL_FN:-${DIAG_TABLE_FN}}${dot_ccpp_phys_suite_or_null}" +FIELD_TABLE_TMPL_FN="${FIELD_TABLE_TMPL_FN:-${FIELD_TABLE_FN}}${dot_ccpp_phys_suite_or_null}" +MODEL_CONFIG_TMPL_FN="${MODEL_CONFIG_TMPL_FN:-${MODEL_CONFIG_FN}}" +NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_TMPL_FN:-${NEMS_CONFIG_FN}}" DATA_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DATA_TABLE_TMPL_FN}" DIAG_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DIAG_TABLE_TMPL_FN}" @@ -2429,6 +2440,12 @@ FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} # GLOBAL_VAR_DEFNS_FP='${GLOBAL_VAR_DEFNS_FP}' +DATA_TABLE_FN='${DATA_TABLE_FN}' +DIAG_TABLE_FN='${DIAG_TABLE_FN}' +FIELD_TABLE_FN='${FIELD_TABLE_FN}' +MODEL_CONFIG_FN='${MODEL_CONFIG_FN}' +NEMS_CONFIG_FN='${NEMS_CONFIG_FN}' + DATA_TABLE_TMPL_FN='${DATA_TABLE_TMPL_FN}' DIAG_TABLE_TMPL_FN='${DIAG_TABLE_TMPL_FN}' FIELD_TABLE_TMPL_FN='${FIELD_TABLE_TMPL_FN}' From e99a458ab3215c81504ec5f75b3da728d7729517 Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Mon, 4 Apr 2022 13:34:47 -0600 Subject: [PATCH 147/203] Changes to RRFS 3- and 13-km domains, setup.sh script bug fixes, make_ics task modification, and tweaks to stochastic physics namelist settings (#721) * Modify RRFS North America 3- and 13-km domain configuration and WE2E test. * Change sotyp_from_climo to "true" based on operational RAP grib2 files. * Update for changes to stochastic physics namelist options. * Check for DO_ENSEMBLE="TRUE" when running ensemble verification and turn of VX when running in NCO mode. * Revert to 3-km domain. * Remove commented-out GFDL grid for the RRFS_NA_13km domain * Add RRFS_NA_13km WE2E test * Changes to comments. --- scripts/exregional_make_ics.sh | 2 +- ...cs_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 54 +++++++++++ ...s_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh | 2 +- ush/generate_FV3LAM_wflow.sh | 16 ++-- ush/set_predef_grid_params.sh | 90 +++++-------------- ush/setup.sh | 25 ++++-- 6 files changed, 107 insertions(+), 82 deletions(-) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 8a63b19786..8a4b834453 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -437,7 +437,7 @@ case "${EXTRN_MDL_NAME_ICS}" in # geogrid_file_input_grid="${FIXgsm}/geo_em.d01.nc_RAPX" vgtyp_from_climo=True - sotyp_from_climo=False + sotyp_from_climo=True vgfrc_from_climo=True minmax_vgfrc_from_climo=True lai_from_climo=True diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..9e4cb594b1 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -0,0 +1,54 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_NA_13km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the FV3GFS. +# +# Note that this test also sets various resource parameters for several +# of the rocoto tasks in order to more efficiently run the code on this +# (very large) grid. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_NA_13km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +######################################################################### +# The following code/namelist/workflow setting changes are necessary to # +# run/optimize end-to-end experiments using the 3-km NA grid # +######################################################################### + +# The model should be built in 32-bit mode (64-bit will result in much +# longer run times. + +# Use k_split=2 and n_split=5, the previous namelist values (k_split=4 +# and n_split=5) will result in significantly longer run times. + +NNODES_MAKE_ICS="12" +NNODES_MAKE_LBCS="12" +PPN_MAKE_ICS="4" +PPN_MAKE_LBCS="4" +WTIME_MAKE_LBCS="01:00:00" + +NNODES_RUN_POST="6" +PPN_RUN_POST="12" + +OMP_STACKSIZE_MAKE_ICS="2048m" +OMP_STACKSIZE_RUN_FCST="2048m" + +############################################################################### diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index c088805dec..f743ebb40e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -45,7 +45,7 @@ PPN_MAKE_ICS="4" PPN_MAKE_LBCS="4" WTIME_MAKE_LBCS="01:00:00" -NNODES_RUN_POST="6" +NNODES_RUN_POST="8" PPN_RUN_POST="12" OMP_STACKSIZE_MAKE_ICS="2048m" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index d0f5d3bd4b..83282b7177 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -856,8 +856,9 @@ settings="$settings fi # # Add the relevant tendency-based stochastic physics namelist variables to -# "settings" when running with SPPT, SHUM, or SKEB turned on. Otherwise -# only include an empty "nam_stochy" stanza. +# "settings" when running with SPPT, SHUM, or SKEB turned on. If running +# with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only +# include an empty "nam_stochy" stanza. # settings="$settings 'nam_stochy': {" @@ -881,8 +882,7 @@ if [ "${DO_SHUM}" = "TRUE" ]; then 'shum': ${SHUM_MAG}, 'shum_lscale': ${SHUM_LSCALE}, 'shum_tau': ${SHUM_TSCALE}, - 'shumint': ${SHUM_INT}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," + 'shumint': ${SHUM_INT}," fi if [ "${DO_SKEB}" = "TRUE" ]; then @@ -894,8 +894,12 @@ if [ "${DO_SKEB}" = "TRUE" ]; then 'skebnorm': ${SKEBNORM}, 'skeb_tau': ${SKEB_TSCALE}, 'skebint': ${SKEB_INT}, - 'skeb_vdof': ${SKEB_VDOF}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," + 'skeb_vdof': ${SKEB_VDOF}," +fi + +if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then + settings="$settings + 'new_lscale': ${NEW_LSCALE}," fi settings="$settings }" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index dd664670ab..808dd4c91f 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -1121,60 +1121,16 @@ case ${PREDEF_GRID_NAME} in # "RRFS_NA_13km") -# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then -# -# GFDLgrid_LON_T6_CTR="-106.0" -# GFDLgrid_LAT_T6_CTR="54.0" -# GFDLgrid_STRETCH_FAC="0.63" -# GFDLgrid_RES="384" -# GFDLgrid_REFINE_RATIO="3" -# -# num_margin_cells_T6_left="10" -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) -# -# num_margin_cells_T6_right="10" -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) -# -# num_margin_cells_T6_bottom="10" -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) -# -# num_margin_cells_T6_top="10" -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) -# -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" -# -# DT_ATMOS="50" -# -# LAYOUT_X="14" -# LAYOUT_Y="14" -# BLOCKSIZE="26" -# -# if [ "$QUILTING" = "TRUE" ]; then -# WRTCMP_write_groups="1" -# WRTCMP_write_tasks_per_group="14" -# WRTCMP_output_grid="rotated_latlon" -# WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}" -# WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}" -# WRTCMP_lon_lwr_left="-57.9926" -# WRTCMP_lat_lwr_left="-50.74344" -# WRTCMP_lon_upr_rght="57.99249" -# WRTCMP_lat_upr_rght="50.74344" -# WRTCMP_dlon="0.1218331" -# WRTCMP_dlat="0.121833" -# fi -# -# elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR="-106.0" - ESGgrid_LAT_CTR="54.0" + ESGgrid_LON_CTR="-112.5" + ESGgrid_LAT_CTR="55.0" ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" - ESGgrid_NX="960" - ESGgrid_NY="960" + ESGgrid_NX="912" + ESGgrid_NY="623" ESGgrid_PAZI="0.0" @@ -1190,12 +1146,12 @@ case ${PREDEF_GRID_NAME} in WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group="16" WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-55.82538869" - WRTCMP_lat_lwr_left="-48.57685654" - WRTCMP_lon_upr_rght="55.82538869" - WRTCMP_lat_upr_rght="48.57685654" + WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" + WRTCMP_lon_lwr_left="-61.0" + WRTCMP_lat_lwr_left="-37.0" + WRTCMP_lon_upr_rght="61.0" + WRTCMP_lat_upr_rght="37.0" WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi @@ -1212,35 +1168,35 @@ case ${PREDEF_GRID_NAME} in GRID_GEN_METHOD="ESGgrid" - ESGgrid_LON_CTR=-107.5 - ESGgrid_LAT_CTR=51.5 + ESGgrid_LON_CTR=-112.5 + ESGgrid_LAT_CTR=55.0 ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" - ESGgrid_NX=3640 - ESGgrid_NY=2520 + ESGgrid_NX=3950 + ESGgrid_NY=2700 - ESGgrid_PAZI="-13.0" + ESGgrid_PAZI="0.0" ESGgrid_WIDE_HALO_WIDTH=6 DT_ATMOS="${DT_ATMOS:-36}" - LAYOUT_X="${LAYOUT_X:-18}" # 40 - EMC operational configuration - LAYOUT_Y="${LAYOUT_Y:-36}" # 45 - EMC operational configuration + LAYOUT_X="${LAYOUT_X:-20}" # 40 - EMC operational configuration + LAYOUT_Y="${LAYOUT_Y:-35}" # 45 - EMC operational configuration BLOCKSIZE="${BLOCKSIZE:-28}" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" WRTCMP_write_tasks_per_group="144" WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="-112.0" #${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="48.0" #${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-51.0" - WRTCMP_lat_lwr_left="-33.0" - WRTCMP_lon_upr_rght="51.0" - WRTCMP_lat_upr_rght="33.0" + WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" + WRTCMP_lon_lwr_left="-61.0" + WRTCMP_lat_lwr_left="-37.0" + WRTCMP_lon_upr_rght="61.0" + WRTCMP_lat_upr_rght="37.0" WRTCMP_dlon="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) WRTCMP_dlat="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) fi diff --git a/ush/setup.sh b/ush/setup.sh index cf76d1d2bd..26dd58b17f 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1404,6 +1404,21 @@ fi # #----------------------------------------------------------------------- # +# Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. +# +#----------------------------------------------------------------------- +# +if [ "${DO_ENSEMBLE}" = "FALSE" ] && [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" -o \ + "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then + print_err_msg_exit "\ +Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"${DO_ENSEMBLE}\" + RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" + RUN_TASK_VX_ENSPOINT = \"${RUN_TASK_VX_ENSPOINT}\"" +fi +# +#----------------------------------------------------------------------- +# # Set the full path to the forecast model executable. # #----------------------------------------------------------------------- @@ -1587,8 +1602,7 @@ one above. Reset values are: fi - if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -1607,8 +1621,7 @@ Reset value is:" fi - if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -1627,8 +1640,7 @@ Reset value is:" fi - if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_ENSGRID}" = "FALSE" ]; then + if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ]; then msg=" When RUN_ENVIR is set to \"nco\", it is assumed that the verification @@ -2022,7 +2034,6 @@ SUB_HOURLY_POST is NOT available with Inline Post yet." fi fi - check_var_valid_value "QUILTING" "valid_vals_QUILTING" QUILTING=$(boolify $QUILTING) From e36d3670f9866462870e08137f91af2e5ae0b45d Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Mon, 4 Apr 2022 15:58:03 -0600 Subject: [PATCH 148/203] Adding 25 km tests to Jet/Hera suites. (#718) --- tests/WE2E/machine_suites/hera.txt | 11 ++++++++++- tests/WE2E/machine_suites/jet.txt | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/machine_suites/hera.txt b/tests/WE2E/machine_suites/hera.txt index 35b26e9b1b..9316d68898 100644 --- a/tests/WE2E/machine_suites/hera.txt +++ b/tests/WE2E/machine_suites/hera.txt @@ -1 +1,10 @@ -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/machine_suites/jet.txt b/tests/WE2E/machine_suites/jet.txt index 35b26e9b1b..9316d68898 100644 --- a/tests/WE2E/machine_suites/jet.txt +++ b/tests/WE2E/machine_suites/jet.txt @@ -1 +1,10 @@ -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 +grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha +grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR From 95bf1fc01f91df9c559681fc3b78b34959a5b003 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 11 Apr 2022 09:39:54 -0600 Subject: [PATCH 149/203] Add a small 3km predefined grid over Indianapolis for testing (#725) * Add 3km grid over Indianapolis. This is about 600km x 600km in extent (200 x 200 grid points). It is intended for use in the WE2E tests. * Edit comments. --- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 27 +++++++++++ .../wflow_features/config.new_ESGgrid.sh | 4 +- ush/set_predef_grid_params.sh | 48 +++++++++++++++++++ ush/valid_param_vals.sh | 1 + 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..e11ab393a3 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the SUBCONUS_Ind_3km grid using the GFS_v16 +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="SUBCONUS_Ind_3km" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "18" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh index b56681f549..fc34f4a0c3 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh @@ -2,7 +2,7 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the user +# This test checks the capability of the workflow to have the user # specify a new grid (as opposed to one of the predefined ones in the # workflow) of ESGgrid type. @@ -47,7 +47,7 @@ BLOCKSIZE="13" QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) WRTCMP_output_grid="lambert_conformal" WRTCMP_cen_lon="${ESGgrid_LON_CTR}" WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 808dd4c91f..53aaf7c603 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -279,6 +279,54 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # +# A subconus domain over Indianapolis, Indiana with ~3km cells. This is +# mostly for testing on a 3km grid with a much small number of cells than +# on the full CONUS. +# +#----------------------------------------------------------------------- +# +"SUBCONUS_Ind_3km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR="-86.16" + ESGgrid_LAT_CTR="39.77" + + ESGgrid_DELX="3000.0" + ESGgrid_DELY="3000.0" + + ESGgrid_NX="200" + ESGgrid_NY="200" + + ESGgrid_PAZI="0.0" + + ESGgrid_WIDE_HALO_WIDTH="6" + + DT_ATMOS="${DT_ATMOS:-40}" + + LAYOUT_X="${LAYOUT_X:-5}" + LAYOUT_Y="${LAYOUT_Y:-5}" + BLOCKSIZE="${BLOCKSIZE:-40}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="197" + WRTCMP_ny="195" + WRTCMP_lon_lwr_left="-89.47120417" + WRTCMP_lat_lwr_left="37.07809642" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" + fi + ;; +# +#----------------------------------------------------------------------- +# # The RRFS Alaska domain with ~13km cells. # # Note: diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 1ea3a86b85..a41f8eb86f 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -27,6 +27,7 @@ valid_vals_PREDEF_GRID_NAME=( \ "GSD_HRRR_AK_50km" \ "RRFS_NA_13km" \ "RRFS_NA_3km" \ +"SUBCONUS_Ind_3km" \ ) valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GFS_2017_gfdlmp" \ From 779accc1ac4f93933481f757ace9e2d093da988e Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Tue, 12 Apr 2022 13:40:06 -0600 Subject: [PATCH 150/203] Use Python tool for get_extrnl_mdl_file tasks (#681) These changes hook in the Python-based data ingest tool, replacing the previous scripts that handled this work as part of the get_extrn_mdl_file task. No attempt was made in this PR to replace the NOMADS fetching script with the Python utility, but the NOMADS data location has been added to the data_locations.yml file. The functionality to write the data summary file has also been added to the Python tool to match the capabilities of the existing workflow tools. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 260 +++--- modulefiles/tasks/cheyenne/get_extrn_ics | 5 +- modulefiles/tasks/cheyenne/get_extrn_lbcs | 5 +- modulefiles/tasks/cheyenne/make_grid.local | 9 +- modulefiles/tasks/cheyenne/make_ics.local | 9 +- modulefiles/tasks/cheyenne/make_lbcs.local | 9 +- .../tasks/cheyenne/pylib_regional_workflow | 9 + modulefiles/tasks/cheyenne/run_fcst.local | 9 +- modulefiles/tasks/cheyenne/run_vx.local | 9 +- modulefiles/tasks/hera/get_extrn_ics.local | 1 + modulefiles/tasks/hera/get_extrn_lbcs.local | 1 + modulefiles/tasks/hera/make_grid.local | 4 +- modulefiles/tasks/hera/make_ics.local | 4 +- modulefiles/tasks/hera/make_lbcs.local | 5 +- .../tasks/hera/miniconda_regional_workflow | 5 + modulefiles/tasks/hera/run_fcst.local | 5 +- modulefiles/tasks/jet/get_extrn_ics.local | 1 + modulefiles/tasks/jet/get_extrn_lbcs.local | 1 + modulefiles/tasks/jet/make_grid.local | 4 +- modulefiles/tasks/jet/make_ics.local | 7 +- modulefiles/tasks/jet/make_lbcs.local | 7 +- .../tasks/jet/miniconda_regional_workflow | 5 + modulefiles/tasks/jet/run_fcst.local | 5 +- modulefiles/tasks/orion/get_extrn_ics.local | 1 + modulefiles/tasks/orion/get_extrn_lbcs.local | 1 + modulefiles/tasks/orion/make_grid.local | 5 +- modulefiles/tasks/orion/make_ics.local | 5 +- modulefiles/tasks/orion/make_lbcs.local | 5 +- .../tasks/orion/miniconda_regional_workflow | 5 + modulefiles/tasks/orion/run_fcst.local | 5 +- scripts/exregional_get_extrn_mdl_files.sh | 757 ++---------------- scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 6 +- tests/WE2E/run_WE2E_tests.sh | 56 +- ush/config_defaults.sh | 45 +- ush/get_extrn_mdl_file_dir_info.sh | 684 ---------------- ush/retrieve_data.py | 283 ++++--- ush/templates/FV3LAM_wflow.xml | 2 - ush/templates/data_locations.yml | 71 +- 39 files changed, 451 insertions(+), 1861 deletions(-) create mode 100644 modulefiles/tasks/cheyenne/pylib_regional_workflow create mode 100644 modulefiles/tasks/hera/miniconda_regional_workflow create mode 100644 modulefiles/tasks/jet/miniconda_regional_workflow create mode 100644 modulefiles/tasks/orion/miniconda_regional_workflow delete mode 100755 ush/get_extrn_mdl_file_dir_info.sh mode change 100644 => 100755 ush/retrieve_data.py diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index a452062c5f..31c2963866 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -33,15 +33,6 @@ # #----------------------------------------------------------------------- # -# Source the file defining the function that will be used to set various -# external-model-associated variables. -# -#----------------------------------------------------------------------- -# -. $USHDIR/get_extrn_mdl_file_dir_info.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -72,33 +63,107 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the J-job script for the task that copies/fetches to a local -directory (either from disk or HPSS) the external model files from which -initial or boundary condition files for the FV3 will be generated. +This is the J-job script for the task that copies or fetches external +model files from disk, HPSS, or URL, and stages them for downstream use +to generate initial or lateral boundary conditions for the FV3 model. ========================================================================" + + +# +#----------------------------------------------------------------------- +# +# Check whether the environment variable ICS_OR_LBCS is set to a valid +# value. This variable specifies whether we are getting the external +# model files for the purpose of generating initial conditions (ICs) or +# lateral boundary condtions (LBCs) for the forecast model. +# +#----------------------------------------------------------------------- +# +valid_vals_ICS_OR_LBCS=( "ICS" "LBCS" ) +check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" +# +#----------------------------------------------------------------------- +# +# Set parameters for grabbing either the initial conditions from analysis or +# forecast files of external models, or the lateral boundary conditions +# from external models. This script has been called to do the work for +# one or the other. +# +#----------------------------------------------------------------------- +# +if [ "${ICS_OR_LBCS}" = "ICS" ]; then + time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} + extrn_mdl_name=${EXTRN_MDL_NAME_ICS} + +elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then + time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} + extrn_mdl_name=${EXTRN_MDL_NAME_LBCS} +fi + +# +#----------------------------------------------------------------------- +# +# Set the external model start time +# +#----------------------------------------------------------------------- +# + +hh=${CDATE:8:2} +yyyymmdd=${CDATE:0:8} +extrn_mdl_cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) + # #----------------------------------------------------------------------- # -# Check whether output files from the specified external model (EXTRN_MDL_NAME) -# are available on the specified cycle date and time (CDATE). +# Check whether output files from the specified external model +# (extrn_mdl_name) are available on the specified cycle date and time +# (extrn_mdl_cdate). # #----------------------------------------------------------------------- # -case ${EXTRN_MDL_NAME} in + +function data_unavailable() { + + local name cdate end_date min_max + + name=$1 + cdate=$2 + end_date=$3 + min_max=$4 + + if [ ${min_max} = max ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +later than the last forecast date and time (cdate_max) with this model: + extrn_mdl_name = \"${name}\" + CDATE_max = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" + + elif [ ${min_max} = min ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +earlier than the implementation date of this model: + extrn_mdl_name = \"${name}\" + CDATE_min = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" + fi + + echo ${msg} +} + + +case ${extrn_mdl_name} in "GSMGFS") # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. # this was the first official forecast with the FV3GFS. So we set the # last CDATE for the GSMGFS to the one 6 hours before this. cdate_max="2019061206" - if [ "$CDATE" -gt "$cdate_max" ]; then + if [ "$extrn_mdl_cdate" -gt "$cdate_max" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -later than the last forecast date and time (cdate_max) with this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - cdate_max = \"${cdate_max}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)" fi ;; @@ -106,17 +171,12 @@ later than the last forecast date and time (cdate_max) with this model: # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. # this was the first official forecast with the FV3GFS. However, paral- # lel runs with the FV3GFS go back to 2018121500. So we set the first -# CDATE for the FV3GFS to this date and time. +# extrn_mdl_cdate for the FV3GFS to this date and time. # CDATE_min="2019061212" CDATE_min="2018121500" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -124,14 +184,9 @@ earlier than the implementation date of this model: # Examination of the HPSS archives shows that the RAPX data goes back to # July 01, 2015. CDATE_min="2015070100" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -140,14 +195,9 @@ earlier than the implementation date of this model: # implementation of the first version of the operational HRRR was # September 30, 2014. CDATE_min="2014103000" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -155,141 +205,25 @@ esac # #----------------------------------------------------------------------- # -# Check whether the environment variable ICS_OR_LBCS is set to a valid -# value. This variable specifies whether we are getting the external -# model files for the purpose of generating initial conditions (ICs) or -# lateral boundary condtions (LBCs) for the forecast model. -# -#----------------------------------------------------------------------- -# -valid_vals_ICS_OR_LBCS=( "ICS" "LBCS" ) -check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" -# -#----------------------------------------------------------------------- -# -# Set parameters for grabbing either the initial conditions from analysis or -# forecast files of external models, or the lateral boundary conditions -# from external models. The script has been called to do the work for -# one or the other. -# -#----------------------------------------------------------------------- -# -if [ "${ICS_OR_LBCS}" = "ICS" ]; then - if [ ${EXTRN_MDL_ICS_OFFSET_HRS} -eq 0 ] ; then - anl_or_fcst="ANL" - time_offset_hrs=0 - else - anl_or_fcst="FCST" - time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} - fi -elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - anl_or_fcst="FCST" - time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} -fi -# -#----------------------------------------------------------------------- -# # Create the directory where the exetrnal model files should be stored # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" +extrn_mdl_staging_dir="${CYCLE_DIR}/${extrn_mdl_name}/for_${ICS_OR_LBCS}" mkdir_vrfy -p "${extrn_mdl_staging_dir}" cd_vrfy "${extrn_mdl_staging_dir}" # #----------------------------------------------------------------------- # -# Call the function that sets various external-model-associated variables. -# See the function defintion file for the definitions of these variables. -# -#----------------------------------------------------------------------- -# -get_extrn_mdl_file_dir_info \ - extrn_mdl_name="${EXTRN_MDL_NAME}" \ - anl_or_fcst="${anl_or_fcst}" \ - cdate_FV3LAM="${CDATE}" \ - time_offset_hrs="${time_offset_hrs}" \ - varname_extrn_mdl_cdate="extrn_mdl_cdate" \ - varname_extrn_mdl_lbc_spec_fhrs="extrn_mdl_lbc_spec_fhrs" \ - varname_extrn_mdl_fns_on_disk="extrn_mdl_fns_on_disk" \ - varname_extrn_mdl_fns_in_arcv="extrn_mdl_fns_in_arcv" \ - varname_extrn_mdl_sysdir="extrn_mdl_sysdir" \ - varname_extrn_mdl_arcv_fmt="extrn_mdl_arcv_fmt" \ - varname_extrn_mdl_arcv_fns="extrn_mdl_arcv_fns" \ - varname_extrn_mdl_arcv_fps="extrn_mdl_arcv_fps" \ - varname_extrn_mdl_arcvrel_dir="extrn_mdl_arcvrel_dir" || \ -print_err_msg_exit "\ -Call to function get_extrn_mdl_file_dir_info failed." -# -#----------------------------------------------------------------------- -# -# Set the directory in which to check for the external model files (which -# we refer to here as the "source" directory) to the default one set above -# for the current machine and external model. -# -#----------------------------------------------------------------------- -# -extrn_mdl_source_dir="${extrn_mdl_sysdir}" -# -#----------------------------------------------------------------------- -# -# If the user has specified that the external model files to be used for -# generating ICs or LBCs are staged, then reset extrn_mdl_source_dir to -# the user-specified directory in which these files are staged, and reset -# extrn_mdl_fns_on_disk to the user-specified array containing the names -# of the files. -# -#----------------------------------------------------------------------- -# -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - - if [ "${ICS_OR_LBCS}" = "ICS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_ICS[@]}" )) - elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_LBCS[@]}" )) - fi - - if [ ! -d "${extrn_mdl_source_dir}" ]; then - print_err_msg_exit "\ -The directory extrn_mdl_source_dir containing the user-staged external -model files does not exist: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - fi - -fi -# -#----------------------------------------------------------------------- -# # Call the ex-script for this J-job and pass to it the necessary variables. # #----------------------------------------------------------------------- # -extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -extrn_mdl_fns_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -extrn_mdl_arcv_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fns[@]}" )")" -extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \ - ics_or_lbcs="${ICS_OR_LBCS}" \ - use_user_staged_extrn_files="${USE_USER_STAGED_EXTRN_FILES}" \ extrn_mdl_cdate="${extrn_mdl_cdate}" \ - extrn_mdl_lbc_spec_fhrs="${extrn_mdl_lbc_spec_fhrs_str}" \ - extrn_mdl_fns_on_disk="${extrn_mdl_fns_on_disk_str}" \ - extrn_mdl_fns_in_arcv="${extrn_mdl_fns_in_arcv_str}" \ - extrn_mdl_source_dir="${extrn_mdl_source_dir}" \ + extrn_mdl_name="${extrn_mdl_name}" \ extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \ - extrn_mdl_arcv_fmt="${extrn_mdl_arcv_fmt}" \ - extrn_mdl_arcv_fns="${extrn_mdl_arcv_fns_str}" \ - extrn_mdl_arcv_fps="${extrn_mdl_arcv_fps_str}" \ - extrn_mdl_arcvrel_dir="${extrn_mdl_arcvrel_dir}" || \ + time_offset_hrs=${time_offset_hrs} || print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/modulefiles/tasks/cheyenne/get_extrn_ics b/modulefiles/tasks/cheyenne/get_extrn_ics index 58f82dca16..9f7a4e4f73 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_ics +++ b/modulefiles/tasks/cheyenne/get_extrn_ics @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/get_extrn_lbcs b/modulefiles/tasks/cheyenne/get_extrn_lbcs index 58f82dca16..9f7a4e4f73 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_lbcs +++ b/modulefiles/tasks/cheyenne/get_extrn_lbcs @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_grid.local b/modulefiles/tasks/cheyenne/make_grid.local index 4232b86593..2f92a39e57 100644 --- a/modulefiles/tasks/cheyenne/make_grid.local +++ b/modulefiles/tasks/cheyenne/make_grid.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_ics.local b/modulefiles/tasks/cheyenne/make_ics.local index 4232b86593..2f92a39e57 100644 --- a/modulefiles/tasks/cheyenne/make_ics.local +++ b/modulefiles/tasks/cheyenne/make_ics.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_lbcs.local b/modulefiles/tasks/cheyenne/make_lbcs.local index 4232b86593..2f92a39e57 100644 --- a/modulefiles/tasks/cheyenne/make_lbcs.local +++ b/modulefiles/tasks/cheyenne/make_lbcs.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/pylib_regional_workflow b/modulefiles/tasks/cheyenne/pylib_regional_workflow new file mode 100644 index 0000000000..4232b86593 --- /dev/null +++ b/modulefiles/tasks/cheyenne/pylib_regional_workflow @@ -0,0 +1,9 @@ +#%Module +if [module-info mode load] { + system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" +} + +if [module-info mode remove] { + system "deactivate" +} + diff --git a/modulefiles/tasks/cheyenne/run_fcst.local b/modulefiles/tasks/cheyenne/run_fcst.local index 4232b86593..2f92a39e57 100644 --- a/modulefiles/tasks/cheyenne/run_fcst.local +++ b/modulefiles/tasks/cheyenne/run_fcst.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local index 234d799088..8fb3be36da 100644 --- a/modulefiles/tasks/cheyenne/run_vx.local +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -1,11 +1,4 @@ #%Module - -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} +module load pylib_regional_workflow module use /glade/p/ral/jntp/MET/MET_releases/modulefiles module load met/10.0.0 diff --git a/modulefiles/tasks/hera/get_extrn_ics.local b/modulefiles/tasks/hera/get_extrn_ics.local index 9935033fd2..4b0b48cc00 100644 --- a/modulefiles/tasks/hera/get_extrn_ics.local +++ b/modulefiles/tasks/hera/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/get_extrn_lbcs.local b/modulefiles/tasks/hera/get_extrn_lbcs.local index 1919f3355a..477dfb2e40 100644 --- a/modulefiles/tasks/hera/get_extrn_lbcs.local +++ b/modulefiles/tasks/hera/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_grid.local b/modulefiles/tasks/hera/make_grid.local index 011a832c9b..92505cf09c 100644 --- a/modulefiles/tasks/hera/make_grid.local +++ b/modulefiles/tasks/hera/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_ics.local b/modulefiles/tasks/hera/make_ics.local index 011a832c9b..92505cf09c 100644 --- a/modulefiles/tasks/hera/make_ics.local +++ b/modulefiles/tasks/hera/make_ics.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_lbcs.local b/modulefiles/tasks/hera/make_lbcs.local index 011a832c9b..61a3a77250 100644 --- a/modulefiles/tasks/hera/make_lbcs.local +++ b/modulefiles/tasks/hera/make_lbcs.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow b/modulefiles/tasks/hera/miniconda_regional_workflow new file mode 100644 index 0000000000..011a832c9b --- /dev/null +++ b/modulefiles/tasks/hera/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/hera/run_fcst.local b/modulefiles/tasks/hera/run_fcst.local index 011a832c9b..61a3a77250 100644 --- a/modulefiles/tasks/hera/run_fcst.local +++ b/modulefiles/tasks/hera/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_ics.local b/modulefiles/tasks/jet/get_extrn_ics.local index 9935033fd2..4b0b48cc00 100644 --- a/modulefiles/tasks/jet/get_extrn_ics.local +++ b/modulefiles/tasks/jet/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_lbcs.local b/modulefiles/tasks/jet/get_extrn_lbcs.local index 1919f3355a..477dfb2e40 100644 --- a/modulefiles/tasks/jet/get_extrn_lbcs.local +++ b/modulefiles/tasks/jet/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_grid.local b/modulefiles/tasks/jet/make_grid.local index 011a832c9b..92505cf09c 100644 --- a/modulefiles/tasks/jet/make_grid.local +++ b/modulefiles/tasks/jet/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_ics.local b/modulefiles/tasks/jet/make_ics.local index c3c0f61e55..61a3a77250 100644 --- a/modulefiles/tasks/jet/make_ics.local +++ b/modulefiles/tasks/jet/make_ics.local @@ -1,7 +1,2 @@ #%Module -module load wgrib2/2.0.8 - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_lbcs.local b/modulefiles/tasks/jet/make_lbcs.local index c3c0f61e55..61a3a77250 100644 --- a/modulefiles/tasks/jet/make_lbcs.local +++ b/modulefiles/tasks/jet/make_lbcs.local @@ -1,7 +1,2 @@ #%Module -module load wgrib2/2.0.8 - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/miniconda_regional_workflow b/modulefiles/tasks/jet/miniconda_regional_workflow new file mode 100644 index 0000000000..011a832c9b --- /dev/null +++ b/modulefiles/tasks/jet/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/run_fcst.local b/modulefiles/tasks/jet/run_fcst.local index 011a832c9b..61a3a77250 100644 --- a/modulefiles/tasks/jet/run_fcst.local +++ b/modulefiles/tasks/jet/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_ics.local b/modulefiles/tasks/orion/get_extrn_ics.local index a9d5b44121..655aa30a2c 100644 --- a/modulefiles/tasks/orion/get_extrn_ics.local +++ b/modulefiles/tasks/orion/get_extrn_ics.local @@ -3,4 +3,5 @@ ############################################################# module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_lbcs.local b/modulefiles/tasks/orion/get_extrn_lbcs.local index 09f37151af..a05a15faa9 100644 --- a/modulefiles/tasks/orion/get_extrn_lbcs.local +++ b/modulefiles/tasks/orion/get_extrn_lbcs.local @@ -4,3 +4,4 @@ module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 4de2a79ca9..92505cf09c 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,6 +1,3 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 3703a9ba17..61a3a77250 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 3703a9ba17..61a3a77250 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow b/modulefiles/tasks/orion/miniconda_regional_workflow new file mode 100644 index 0000000000..3703a9ba17 --- /dev/null +++ b/modulefiles/tasks/orion/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3/3.8 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 3703a9ba17..61a3a77250 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index ac5127eb8b..fc02e3b996 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -42,9 +42,10 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the ex-script for the task that copies/fetches to a local directory -either from disk or HPSS) the external model files from which initial or -boundary condition files for the FV3 will be generated. +This is the ex-script for the task that copies or fetches external model +input data from disk, HPSS, or a URL, and stages them to the +workflow-specified location so that they may be used to generate initial +or lateral boundary conditions for the FV3. ========================================================================" # #----------------------------------------------------------------------- @@ -56,18 +57,10 @@ boundary condition files for the FV3 will be generated. #----------------------------------------------------------------------- # valid_args=( \ -"ics_or_lbcs" \ -"use_user_staged_extrn_files" \ "extrn_mdl_cdate" \ -"extrn_mdl_lbc_spec_fhrs" \ -"extrn_mdl_fns_on_disk" \ -"extrn_mdl_fns_in_arcv" \ -"extrn_mdl_source_dir" \ +"extrn_mdl_name" \ "extrn_mdl_staging_dir" \ -"extrn_mdl_arcv_fmt" \ -"extrn_mdl_arcv_fns" \ -"extrn_mdl_arcv_fps" \ -"extrn_mdl_arcvrel_dir" \ +"time_offset_hrs" \ ) process_args valid_args "$@" # @@ -80,715 +73,105 @@ process_args valid_args "$@" #----------------------------------------------------------------------- # print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Set num_files_to_copy to the number of external model files that need -# to be copied or linked to from/at a location on disk. Then set -# extrn_mdl_fps_on_disk to the full paths of the external model files -# on disk. -# -#----------------------------------------------------------------------- -# -num_files_to_copy="${#extrn_mdl_fns_on_disk[@]}" -prefix="${extrn_mdl_source_dir}/" -extrn_mdl_fps_on_disk=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) -# -#----------------------------------------------------------------------- -# -# Loop through the list of external model files and check whether they -# all exist on disk. The counter num_files_found_on_disk keeps track of -# the number of external model files that were actually found on disk in -# the directory specified by extrn_mdl_source_dir. -# -# If the location extrn_mdl_source_dir is a user-specified directory -# (i.e. if use_user_staged_extrn_files is set to "TRUE"), then if/when we -# encounter the first file that does not exist, we exit the script with -# an error message. If extrn_mdl_source_dir is a system directory (i.e. -# if use_user_staged_extrn_files is not set to "TRUE"), then if/when we -# encounter the first file that does not exist or exists but is younger -# than a certain age, we break out of the loop and try to fetch all the -# necessary external model files from HPSS. The age cutoff is to ensure -# that files are not still being written to. -# -#----------------------------------------------------------------------- -# -num_files_found_on_disk="0" -min_age="5" # Minimum file age, in minutes. - -for fp in "${extrn_mdl_fps_on_disk[@]}"; do - # - # If the external model file exists, then... - # - if [ -f "$fp" ]; then - # - # Increment the counter that keeps track of the number of external - # model files found on disk and print out an informational message. - # - num_files_found_on_disk=$(( num_files_found_on_disk+1 )) - print_info_msg " -File fp exists on disk: - fp = \"$fp\"" - # - # If we are NOT searching for user-staged external model files, then - # we also check that the current file is at least min_age minutes old. - # If not, we try searching for all the external model files on HPSS. - # - if [ "${use_user_staged_extrn_files}" != "TRUE" ]; then - - if [ $( find "$fp" -mmin +${min_age} ) ]; then - - print_info_msg " -File fp is older than the minimum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes" - - else - - print_info_msg " -File fp is NOT older than the minumum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - - fi - # - # If the external model file does not exist, then... - # - else - # - # If an external model file is not found and we are searching for it - # in a user-specified directory, print out an error message and exit. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - - print_err_msg_exit "\ -File fp does NOT exist on disk: - fp = \"$fp\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - # - # If an external model file is not found and we are searching for it - # in a system directory, give up on the system directory and try instead - # to get all the external model files from HPSS. - # - else - - print_info_msg " -File fp does NOT exist on disk: - fp = \"$fp\" -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - fi - -done -# -#----------------------------------------------------------------------- -# -# Set the variable (data_src) that determines the source of the external -# model files (either disk or HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${num_files_found_on_disk}" -eq "${num_files_to_copy}" ]; then - data_src="disk" -else - data_src="HPSS" -fi -if [ ${NOMADS} == "TRUE" ]; then - data_src="online" -fi # #----------------------------------------------------------------------- # -# If the source of the external model files is "disk", copy the files -# from the source directory on disk to a staging directory. +# Set up variables for call to retrieve_data.py # #----------------------------------------------------------------------- # -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" - -if [ "${data_src}" = "disk" ]; then - - if [ "${RUN_ENVIR}" = "nco" ]; then - - print_info_msg " -Creating links in staging directory (extrn_mdl_staging_dir) to external -model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str}" - - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - +set -x +if [ "${ICS_OR_LBCS}" = "ICS" ]; then + if [ ${time_offset_hrs} -eq 0 ] ; then + anl_or_fcst="anl" else - - # - # If the external model files are user-staged, then simply link to - # them. Otherwise, if they are on the system disk, copy them to the - # staging directory. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - print_info_msg " -Creating symlinks in the staging directory (extrn_mdl_staging_dir) to the -external model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - else - print_info_msg " -Copying external model files on disk (extrn_mdl_fns_on_disk) from source -directory (extrn_mdl_source_dir) to staging directory (extrn_mdl_staging_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir} - fi - + anl_or_fcst="fcst" fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating initial conditions and surface fields for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating lateral boundary conditions for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - + fcst_hrs=${time_offset_hrs} + file_names=${EXTRN_MDL_FILES_ICS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_ICS fi -# -#----------------------------------------------------------------------- -# -# If the source of the external model files is "HPSS", fetch them from -# HPSS. -# -#----------------------------------------------------------------------- -# -elif [ "${data_src}" = "HPSS" ]; then -# -#----------------------------------------------------------------------- -# -# Set extrn_mdl_fps_in_arcv to the full paths within the archive files of -# the external model files. -# -#----------------------------------------------------------------------- -# - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps_in_arcv=( "${extrn_mdl_fns_in_arcv[@]/#/$prefix}" ) - - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - - print_info_msg " -Fetching external model files from HPSS. The full paths to these files -in the archive file(s) (extrn_mdl_fps_in_arcv), the archive files on HPSS -in which these files are stored (extrn_mdl_arcv_fps), and the staging -directory to which they will be copied (extrn_mdl_staging_dir) are: - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" -# -#----------------------------------------------------------------------- -# -# Get the number of archive files to consider. -# -#----------------------------------------------------------------------- -# - num_arcv_files="${#extrn_mdl_arcv_fps[@]}" -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# tar format. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcv_fmt}" = "tar" ]; then -# -#----------------------------------------------------------------------- -# -# Loop through the set of archive files specified in extrn_mdl_arcv_fps -# and extract a subset of the specified external model files from each. -# -#----------------------------------------------------------------------- -# - num_files_to_extract="${#extrn_mdl_fps_in_arcv[@]}" - - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - - narcv_formatted=$( printf "%02d" $narcv ) - arcv_fp="${extrn_mdl_arcv_fps[$narcv]}" -# -# Before trying to extract (a subset of) the external model files from -# the current tar archive file (which is on HPSS), create a list of those -# external model files that are stored in the current tar archive file. -# For this purpose, we first use the "htar -tvf" command to list all the -# external model files that are in the current archive file and store the -# result in a log file. (This command also indirectly checks whether the -# archive file exists on HPSS.) We then grep this log file for each -# external model file and create a list containing only those external -# model files that exist in the current archive. -# -# Note that the "htar -tvf" command will fail if the tar archive file -# itself doesn't exist on HPSS, but it won't fail if any of the external -# model file names passed to it don't exist in the archive file. In the -# latter case, the missing files' names simply won't appear in the log -# file. -# - htar_log_fn="log.htar_tvf.${narcv_formatted}" - htar -tvf ${arcv_fp} ${extrn_mdl_fps_in_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file list operation (\"htar -tvf ...\") failed. Check the log file -htar_log_fn in the staging directory (extrn_mdl_staging_di)r for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - i=0 - files_in_crnt_arcv=() - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# grep -n ${extrn_mdl_fp} ${htar_log_fn} 2>&1 && { \ - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - files_in_crnt_arcv[$i]="${extrn_mdl_fp}"; \ - i=$((i+1)); \ - } - done -# -# If none of the external model files were found in the current archive -# file, print out an error message and exit. -# - num_files_in_crnt_arcv=${#files_in_crnt_arcv[@]} - if [ ${num_files_in_crnt_arcv} -eq 0 ]; then - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - print_err_msg_exit "\ -The current archive file (arcv_fp) does not contain any of the external -model files listed in extrn_mdl_fps_in_arcv: - arcv_fp = \"${arcv_fp}\" - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} -The archive file should contain at least one external model file; otherwise, -it would not be needed." - fi -# -# Extract from the current tar archive file on HPSS all the external model -# files that exist in that archive file. Also, save the output of the -# "htar -xvf" command in a log file for debugging (if necessary). -# - htar_log_fn="log.htar_xvf.${narcv_formatted}" - htar -xvf ${arcv_fp} ${files_in_crnt_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file extract operation (\"htar -xvf ...\") failed. Check the log -file htar_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" -# -# Note that the htar file extract operation above may return with a 0 -# exit code (success) even if one or more (or all) external model files -# that it is supposed to contain were not extracted. The names of those -# files that were not extracted will not be listed in the log file. Thus, -# we now check whether the log file contains the name of each external -# model file that should have been extracted. If any are missing, we -# print out a message and exit the script because initial condition and -# surface field files needed by FV3 cannot be generated without all the -# external model files. -# - for fp in "${files_in_crnt_arcv[@]}"; do -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - fp=${fp#/} - - grep -n "${fp}" "${htar_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp not extracted from tar archive file arcv_fp: - arcv_fp = \"${arcv_fp}\" - fp = \"$fp\" -Check the log file htar_log_fn in the staging directory (extrn_mdl_staging_dir) -for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - done - - done -# -#----------------------------------------------------------------------- -# -# For each external model file that was supposed to have been extracted -# from the set of specified archive files, loop through the extraction -# log files and check that it appears exactly once in one of the log files. -# If it doesn't appear at all, then it means that file was not extracted, -# and if it appears more than once, then something else is wrong. In -# either case, print out an error message and exit. -# -#----------------------------------------------------------------------- -# - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - extrn_mdl_fp=${extrn_mdl_fp#/} - - num_occurs=0 - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - narcv_formatted=$( printf "%02d" $narcv ) - htar_log_fn="log.htar_xvf.${narcv_formatted}" - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - num_occurs=$((num_occurs+1)); \ - } - done - - if [ ${num_occurs} -eq 0 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) does not appear in any of -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -Thus, it was not extracted, likely because it doesn't exist in any of the -archive files." - elif [ ${num_occurs} -gt 1 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) appears more than once in -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -The number of times it occurs in the log files is: - num_occurs = ${num_occurs} -Thus, it was extracted from more than one archive file, with the last one -that was extracted overwriting all previous ones. This should normally -not happen." - fi - - done -# -#----------------------------------------------------------------------- -# -# If extrn_mdl_arcvrel_dir is not set to the current directory (i.e. it -# is not equal to "."), then the htar command will have created the -# subdirectory "./${extrn_mdl_arcvrel_dir}" under the current directory -# and placed the extracted files there. In that case, we move these -# extracted files back to the current directory and then remove the -# subdirectory created by htar. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcvrel_dir}" != "." ]; then -# -# The code below works if extrn_mdl_arcvrel_dir starts with a "/" or a -# "./", which are the only case encountered thus far. The code may have -# to be modified to accomodate other cases. -# - if [ "${extrn_mdl_arcvrel_dir:0:1}" = "/" ] || \ - [ "${extrn_mdl_arcvrel_dir:0:2}" = "./" ]; then -# -# Strip the "/" or "./" from the beginning of extrn_mdl_arcvrel_dir to -# obtain the relative directory from which to move the extracted files -# to the current directory. Then move the files. -# - rel_dir=$( printf "%s" "${extrn_mdl_arcvrel_dir}" | \ - $SED -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) - mv_vrfy ${rel_dir}/* . -# -# Get the first subdirectory in rel_dir, i.e. the subdirectory before the -# first forward slash. This is the subdirectory that we want to remove -# since it no longer contains any files (only subdirectories). Then remove -# it. -# - subdir_to_remove=$( printf "%s" "${rel_dir}" | \ - $SED -r 's%^([^/]*)(.*)%\1%' ) - rm_vrfy -rf ./${subdir_to_remove} -# -# If extrn_mdl_arcvrel_dir does not start with a "/" (and it is not -# equal to "."), then print out an error message and exit. -# - else - - print_err_msg_exit "\ -The archive-relative directory specified by extrn_mdl_arcvrel_dir [i.e. -the directory \"within\" the tar file(s) listed in extrn_mdl_arcv_fps] is -not the current directory (i.e. it is not \".\"), and it does not start -with a \"/\" or a \"./\": - extrn_mdl_arcvrel_dir = \"${extrn_mdl_arcvrel_dir}\" - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} -This script must be modified to account for this case." - - fi - - fi -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# zip format. -# -#----------------------------------------------------------------------- -# - elif [ "${extrn_mdl_arcv_fmt}" = "zip" ]; then -# -#----------------------------------------------------------------------- -# -# For archive files that are in "zip" format files, the array extrn_mdl_arcv_fps -# containing the list of archive files should contain only one element, -# i.e. there should be only one archive file to consider. Check for this. -# If this ever changes (e.g. due to the way an external model that uses -# the "zip" format archives its output files on HPSS), the code below must -# be modified to loop over all archive files. -# -#----------------------------------------------------------------------- -# - if [ "${num_arcv_files}" -gt 1 ]; then - print_err_msg_exit "\ -Currently, this script is coded to handle only one archive file if the -archive file format is specified to be \"zip\", but the number of archive -files (num_arcv_files) passed to this script is greater than 1: - extrn_mdl_arcv_fmt = \"${extrn_mdl_arcv_fmt}\" - num_arcv_files = ${num_arcv_files} -Please modify the script to handle more than one \"zip\" archive file. -Note that code already exists in this script that can handle multiple -archive files if the archive file format is specified to be \"tar\", so -that can be used as a guide for the \"zip\" case." - else - arcv_fn="${extrn_mdl_arcv_fns[0]}" - arcv_fp="${extrn_mdl_arcv_fps[0]}" - fi -# -#----------------------------------------------------------------------- -# -# Fetch the zip archive file from HPSS. -# -#----------------------------------------------------------------------- -# - hsi_log_fn="log.hsi_get" - hsi get "${arcv_fp}" >& ${hsi_log_fn} || \ - print_err_msg_exit "\ -hsi file get operation (\"hsi get ...\") failed. Check the log file -hsi_log_fn in the staging directory (extrn_mdl_staging_dir) for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - hsi_log_fn = \"${hsi_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# List the contents of the zip archive file and save the result in a log -# file. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip_lv" - unzip -l -v ${arcv_fn} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip operation to list the contents of the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir) failed. Check the log -file unzip_log_fn in that directory for details: - arcv_fn = \"${arcv_fn}\" - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# Check that the log file from the unzip command above contains the name -# of each external model file. If any are missing, then the corresponding -# files are not in the zip file and thus cannot be extracted. In that -# case, print out a message and exit the script because initial condition -# and surface field files for the FV3-LAM cannot be generated without all -# the external model files. -# -#----------------------------------------------------------------------- -# - for fp in "${extrn_mdl_fps_in_arcv[@]}"; do - grep -n "${fp}" "${unzip_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp does not exist in the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir). Check the log file -unzip_log_fn in that directory for the contents of the zip archive: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - arcv_fn = \"${arcv_fn}\" - fp = \"$fp\" - unzip_log_fn = \"${unzip_log_fn}\"" - done -# -#----------------------------------------------------------------------- -# -# Extract the external model files from the zip file on HPSS. Note that -# the -o flag to unzip is needed to overwrite existing files. Otherwise, -# unzip will wait for user input as to whether the existing files should -# be overwritten. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip" - unzip -o "${arcv_fn}" ${extrn_mdl_fps_in_arcv[@]} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip file extract operation (\"unzip -o ...\") failed. Check the log -file unzip_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -# NOTE: -# If extrn_mdl_arcvrel_dir is not empty, the unzip command above will -# create a subdirectory under extrn_mdl_staging_dir and place the external -# model files there. We have not encountered this for the RAP and HRRR -# models, but it may happen for other models in the future. In that case, -# extra code must be included here to move the external model files from -# the subdirectory up to extrn_mdl_staging_dir and then the subdirectory -# (analogous to what is done above for the case of extrn_mdl_arcv_fmt set -# to "tar". -# - + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} + +elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then + anl_or_fcst="fcst" + first_time=$((time_offset_hrs + LBC_SPEC_INTVL_HRS)) + last_time=$((time_offset_hrs + FCST_LEN_HRS)) + fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}" + file_names=${EXTRN_MDL_FILES_LBCS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_LBCS fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -External model files needed for generating initial condition and surface -fields for the FV3-LAM successfully fetched from HPSS!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} +fi - print_info_msg " -======================================================================== -External model files needed for generating lateral boundary conditions -on the halo of the FV3-LAM's regional grid successfully fetched from -HPSS!!! +data_stores="hpss aws" -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" +yyyymmddhh=${extrn_mdl_cdate:0:10} +yyyy=${yyyymmddhh:0:4} +yyyymm=${yyyymmddhh:0:6} +yyyymmdd=${yyyymmddhh:0:8} +mm=${yyyymmddhh:4:2} +dd=${yyyymmddhh:6:2} +hh=${yyyymmddhh:8:2} - fi - -elif [ "${data_src}" = "online" ]; then - print_info_msg " -======================================================================== -getting data from online nomads data sources -========================================================================" +input_file_path=$(eval echo ${input_file_path}) # #----------------------------------------------------------------------- # -# Set extrn_mdl_fps to the full paths within the archive files of the -# external model output files. +# Set up optional flags for calling retrieve_data.py # #----------------------------------------------------------------------- # - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) +additional_flags="" - extrn_mdl_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_fps[@]}" )")" - print_info_msg " -Getting external model files from nomads: - extrn_mdl_fps= ${extrn_mdl_fps_str}" - - num_files_to_extract="${#extrn_mdl_fps[@]}" - wget_LOG_FN="log.wget.txt" - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - cp ../../../${extrn_mdl_fps[$nfile]} . || \ - print_err_msg_exit "\ - onlie file ${extrn_mdl_fps[$nfile]} not found." - done +if [ -n "${file_type:-}" ] ; then + additional_flags="$additional_flags \ + --file_type ${file_type}" +fi +if [ -n "${file_names:-}" ] ; then + additional_flags="$additional_flags \ + --file_templates ${file_names[@]}" +fi +if [ -n "${input_file_path:-}" ] ; then + data_stores="disk hpss aws" + additional_flags="$additional_flags \ + --input_file_path ${input_file_path}" fi + # #----------------------------------------------------------------------- # -# Create a variable definitions file (a shell script) and save in it the -# values of several external-model-associated variables generated in this -# script that will be needed by downstream workflow tasks. +# Call ush script to retrieve files # #----------------------------------------------------------------------- # -if [ "${ics_or_lbcs}" = "ICS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" -elif [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" -fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${extrn_mdl_var_defns_fn}" -check_for_preexist_dir_file "${extrn_mdl_var_defns_fp}" "delete" +cmd=" +python3 -u ${USHDIR}/retrieve_data.py \ + --debug \ + --anl_or_fcst ${anl_or_fcst} \ + --config ${USHDIR}/templates/data_locations.yml \ + --cycle_date ${extrn_mdl_cdate} \ + --data_stores ${data_stores} \ + --external_model ${extrn_mdl_name} \ + --fcst_hrs ${fcst_hrs[@]} \ + --output_path ${extrn_mdl_staging_dir} \ + --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ + $additional_flags" -if [ "${data_src}" = "disk" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -elif [ "${data_src}" = "HPSS" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -elif [ "${data_src}" = "online" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -fi - -settings="\ -DATA_SRC=\"${data_src}\" -EXTRN_MDL_CDATE=\"${extrn_mdl_cdate}\" -EXTRN_MDL_STAGING_DIR=\"${extrn_mdl_staging_dir}\" -EXTRN_MDL_FNS=${extrn_mdl_fns_str}" -# -# If the external model files obtained above were for generating LBCS (as -# opposed to ICs), then add to the external model variable definitions -# file the array variable EXTRN_MDL_LBC_SPEC_FHRS containing the forecast -# hours at which the lateral boundary conditions are specified. -# -if [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" - settings="$settings -EXTRN_MDL_LBC_SPEC_FHRS=${extrn_mdl_lbc_spec_fhrs_str}" -fi +$cmd || print_err_msg_exit "\ +Call to retrieve_data.py failed with a non-zero exit status. -{ cat << EOM >> ${extrn_mdl_var_defns_fp} -$settings -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to create a variable definitions file associated -with the external model from which to generate ${ics_or_lbcs} returned with a -nonzero status. The full path to this variable definitions file is: - extrn_mdl_var_defns_fp = \"${extrn_mdl_var_defns_fp}\"" +The command was: +${cmd} +" # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 8a4b834453..fcfefd6462 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -111,7 +111,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/for_ICS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_ICS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index e1d40fb88c..70ba511763 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -109,7 +109,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- @@ -358,12 +358,12 @@ fi # #----------------------------------------------------------------------- # -num_fhrs="${#EXTRN_MDL_LBC_SPEC_FHRS[@]}" +num_fhrs="${#EXTRN_MDL_FHRS[@]}" for (( i=0; i<${num_fhrs}; i++ )); do # # Get the forecast hour of the external model. # - fhr="${EXTRN_MDL_LBC_SPEC_FHRS[$i]}" + fhr="${EXTRN_MDL_FHRS[$i]}" # # Set external model output file name and file type/format. Note that # these are now inputs into chgres_cube. diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index c162a51004..f3040208ed 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -941,26 +941,21 @@ machine (MACHINE): MACHINE= \"${MACHINE}\"" fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" - fi - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) - fi - elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_ICS" + else + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" fi # # Make sure that the forecast length is evenly divisible by the interval @@ -976,32 +971,15 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" fi - lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) - EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) - fi - elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - fi - - expt_config_str=${expt_config_str}" + expt_config_str="${expt_config_str} # # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" -EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" -EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" +EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS} +EXTRN_MDL_FILES_ICS=( ${EXTRN_MDL_FILES_ICS[@]} ) +EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS} +EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} )" fi # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index be96316d75..145d397ff3 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -387,19 +387,12 @@ DOT_OR_USCORE="_" # to each workflow task) in order to make all the experiment variables # available in those scripts. # -# EXTRN_MDL_ICS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which ICs are generated. This -# file is created by the GET_EXTRN_ICS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. -# -# EXTRN_MDL_LBCS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which LBCs are generated. This -# file is created by the GET_EXTRN_LBCS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. +# EXTRN_MDL_VAR_DEFNS_FN: +# Name of file (a shell script) containing the defintions of variables +# associated with the external model from which ICs or LBCs are generated. This +# file is created by the GET_EXTRN_*_TN task because the values of the variables +# it contains are not known before this task runs. The file is then sourced by +# the MAKE_ICS_TN and MAKE_LBCS_TN tasks. # # WFLOW_LAUNCH_SCRIPT_FN: # Name of the script that can be used to (re)launch the experiment's rocoto @@ -429,8 +422,7 @@ NEMS_CONFIG_TMPL_FN="" FCST_MODEL="ufs-weather-model" WFLOW_XML_FN="FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN="var_defns.sh" -EXTRN_MDL_ICS_VAR_DEFNS_FN="extrn_mdl_ics_var_defns.sh" -EXTRN_MDL_LBCS_VAR_DEFNS_FN="extrn_mdl_lbcs_var_defns.sh" +EXTRN_MDL_VAR_DEFNS_FN="extrn_mdl_var_defns.sh" WFLOW_LAUNCH_SCRIPT_FN="launch_FV3LAM_wflow.sh" WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # @@ -704,9 +696,18 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # set to "FALSE". # # EXTRN_MDL_FILES_ICS: -# Array containing the names of the files to search for in the directory -# specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not used -# if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# Array containing templates of the names of the files to search for in +# the directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This +# variable is not used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# A single template should be used for each model file type that is +# meant to be used. You may use any of the Python-style templates +# allowed in the ush/retrieve_data.py script. To see the full list of +# supported templates, run that script with a -h option. Here is an example of +# setting FV3GFS nemsio input files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio \ +# gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio ) +# Or for FV3GFS grib files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} ) # # EXTRN_MDL_SOURCE_BASEDIR_LBCS: # Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. @@ -717,10 +718,10 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES="FALSE" -EXTRN_MDL_SOURCE_BASEDIR_ICS="/base/dir/containing/user/staged/extrn/mdl/files/for/ICs" -EXTRN_MDL_FILES_ICS=( "ICS_file1" "ICS_file2" "..." ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/base/dir/containing/user/staged/extrn/mdl/files/for/LBCs" -EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) +EXTRN_MDL_SOURCE_BASEDIR_ICS="" +EXTRN_MDL_FILES_ICS="" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="" +EXTRN_MDL_FILES_LBCS="" # #----------------------------------------------------------------------- # diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh deleted file mode 100755 index aab64c2dc1..0000000000 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ /dev/null @@ -1,684 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to obtain information (e.g. -# output file names, system and mass store file and/or directory names) -# for a specified external model, analysis or forecast, and cycle date. -# See the usage statement below for this function should be called and -# the definitions of the input parameters. -# -#----------------------------------------------------------------------- -# - -usage () { - -echo " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} \ - extrn_mdl_name \ - anl_or_fcst \ - cdate_FV3LAM \ - time_offset_hrs \ - varname_extrn_mdl_cdate \ - varname_extrn_mdl_lbc_spec_fhrs \ - varname_extrn_mdl_fns \ - varname_extrn_mdl_sysdir \ - varname_extrn_mdl_arcv_fmt \ - varname_extrn_mdl_arcv_fns \ - varname_extrn_mdl_arcv_fps \ - varname_extrn_mdl_arcvrel_dir - -where the arguments are defined as follows: - - extrn_mdl_name: - Name of the external model, i.e. the name of the model providing the - fields from which files containing initial conditions, surface fields, - and/or lateral boundary conditions for the FV3-LAM will be generated. - - anl_or_fcst: - Flag that specifies whether the external model files we are interested - in obtaining are analysis or forecast files. - - cdate_FV3LAM: - The cycle date and time (hours only) for which we want to obtain file - and directory information. This has the form YYYYMMDDHH, where YYYY - is the four-digit starting year of the cycle, MM is the two-digit - month, DD is the two-digit day of the month, and HH is the two-digit - hour of day. - - time_offset_hrs: - The number of hours by which to shift back in time the start time of - the external model forecast from the specified cycle start time of the - FV3-LAM (cdate_FV3LAM). When getting directory and file information on - external model analysis files, this is normally set to 0. When get- - ting directory and file information on external model forecast files, - this may be set to a nonzero value to obtain information for an exter- - nal model run that started time_offset_hrs hours before cdate_FV3LAM - (instead of exactly at cdate_FV3LAM). Note that in this case, the - forecast hours (relative to the external model run's start time) at - which the lateral boundary conditions will be updated must be shifted - forward by time_offset_hrs hours relative to those for the FV3-LAM in - order to make up for the backward-in-time shift in the starting time - of the external model. - - varname_extrn_mdl_cdate: - Name of the global variable that will contain the starting date and - hour of the external model run. - - varname_extrn_mdl_lbc_spec_fhrs: - Name of the global variable that will contain the forecast hours (re- - lative to the starting time of the external model run, which is earli- - er than that of the FV3-LAM by time_offset_hrs hours) at which lateral - boundary condition (LBC) output files are obtained from the external - model (and will be used to update the LBCs of the FV3-LAM). - - varname_extrn_mdl_fns_on_disk: - Name of the global variable that will contain the expected names of - the external model output files on disk. - - varname_extrn_mdl_fns_in_arcv: - Name of the global variable that will contain the expected names of - the external model output files on NOAA HPSS. - - varname_extrn_mdl_sysdir: - Name of the global variable that will contain the system directory in - which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fmt: - Name of the global variable that will contain the format of the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcv_fns: - Name of the global variable that will contain the name of the archive - file on HPSS in which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fps: - Name of the global variable that will contain the full path to the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcvrel_dir: - Name of the global variable that will contain the archive-relative di- - rectory, i.e. the directory \"inside\" the archive file in which the ex- - ternal model output files may be stored. -" -} - -function quit_unless_user_spec_data() { - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - fi -} - -function get_extrn_mdl_file_dir_info() { - - { save_shell_opts; set -u +x; } > /dev/null 2>&1 - - local func_name="${FUNCNAME[0]}" - # - #----------------------------------------------------------------------- - # - # Specify the set of valid argument names for this script/function. Then - # process the arguments provided to this script/function (which should - # consist of a set of name-value pairs of the form arg1="value1", etc). - # - #----------------------------------------------------------------------- - # - local valid_args=( \ - "extrn_mdl_name" \ - "anl_or_fcst" \ - "cdate_FV3LAM" \ - "time_offset_hrs" \ - "varname_extrn_mdl_cdate" \ - "varname_extrn_mdl_lbc_spec_fhrs" \ - "varname_extrn_mdl_fns_on_disk" \ - "varname_extrn_mdl_fns_in_arcv" \ - "varname_extrn_mdl_sysdir" \ - "varname_extrn_mdl_arcv_fmt" \ - "varname_extrn_mdl_arcv_fns" \ - "varname_extrn_mdl_arcv_fps" \ - "varname_extrn_mdl_arcvrel_dir" \ - ) - process_args valid_args "$@" - - if [ "$#" -ne "13" ]; then - print_err_msg_exit $(usage) - fi - - # - #----------------------------------------------------------------------- - # - # For debugging purposes, print out values of arguments passed to this - # script/function. Note that these will be printed out only if VERBOSE - # is set to TRUE. - # - #----------------------------------------------------------------------- - # - print_input_args valid_args - - # - #----------------------------------------------------------------------- - # - # Declare additional local variables. - # - #----------------------------------------------------------------------- - # - local yyyy yy mm dd hh mn yyyymmdd ddd \ - lbc_spec_fhrs i num_fhrs \ - fcst_hhh fcst_hh fcst_mn \ - prefix suffix fns fns_on_disk fns_in_arcv \ - sysbasedir sysdir \ - arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir - - anl_or_fcst=$(echo_uppercase $anl_or_fcst) - valid_vals_anl_or_fcst=( "ANL" "FCST" ) - check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" - # - #----------------------------------------------------------------------- - # - # Set cdate to the start time for the external model being used. - # - #----------------------------------------------------------------------- - # - hh=${cdate_FV3LAM:8:2} - yyyymmdd=${cdate_FV3LAM:0:8} - - # Adjust time for offset - cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) - - yyyy=${cdate:0:4} - yy=${yyyy:2:4} - mm=${cdate:4:2} - dd=${cdate:6:2} - hh=${cdate:8:2} - mn="00" - yyyymmdd=${cdate:0:8} - # ddd is the Julian day -- not 3 digit day of month - ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) - # - #----------------------------------------------------------------------- - # - # Initialize lbc_spec_fhrs array. Skip the initial time, since it is - # handled separately. - # - #----------------------------------------------------------------------- - # - lbc_spec_fhrs=( "" ) - - if [ "${anl_or_fcst}" = "FCST" ]; then - - lbc_spec_fhrs=( "${LBC_SPEC_FCST_HRS[@]}" ) - - num_fhrs=${#lbc_spec_fhrs[@]} - for (( i=0; i<=$((num_fhrs-1)); i++ )); do - # Add in offset to account for shift in initial time - lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + time_offset_hrs )) - done - - fi - # - #----------------------------------------------------------------------- - # - # The model may be started with a variety of file types from FV3GFS. - # Set that file type now - # - #----------------------------------------------------------------------- - # - - if [ "${anl_or_fcst}" = "ANL" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" - elif [ "${anl_or_fcst}" = "FCST" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" - fi - - # - #----------------------------------------------------------------------- - # - # Generate an array of file names expected from the external model - # Assume that filenames in archive and on disk are the same, unless - # otherwise specified (primarily on Jet). - # - #----------------------------------------------------------------------- - # - declare -a fns_on_disk - declare -a fns_in_arcv - case "${anl_or_fcst}" in - - "ANL") - - fcst_hh="00" - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - ;; - - "FV3GFS") - case "${fv3gfs_file_fmt}" in - "nemsio") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - "grib2") - fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) - ;; - "netcdf") - fns_in_arcv=("gfs.t${hh}z.atmanl.nc" "gfs.t${hh}z.sfcanl.nc") - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - esac - ;; - - "RAP") - ;& # Fall through. RAP and HRRR follow same naming rules - - "HRRR") - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_mn}${fcst_hh}${fcst_mn}" ) - fi - ;; - - "NAM") - fns=( "" ) - fns_in_arcv=( "nam.t${hh}z.awphys${fcst_hh}.tm00.grib2" ) - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names (either on disk or in archive files) have -not yet been specified for this combination of external model (extrn_mdl_name) -and analysis or forecast (anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for ANL files - ;; - - "FCST") - fcst_mn="00" - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - case "${extrn_mdl_name}" in - - "GSMGFS") - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nemsio" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - fn_tmpl="gfs.t${hh}z.pgrb2.0p25.fFHR3" - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nc" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nc" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - fi - ;; - - "RAP") - ;& # Fall through since RAP and HRRR are named the same - - "HRRR") - fn_tmpl="${yy}${ddd}${hh}00FHR200" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}0000FHR2" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - ;; - - "NAM") - fn_tmpl="nam.t${hh}z.awphysFHR2.tm00.grib2" - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names have not yet been specified for this com- -bination of external model (extrn_mdl_name) and analysis or forecast -(anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for FCST files - ;; - esac # End ANL FCST case - - # - # Expand the archive file names for all forecast hours - # - if [ ${anl_or_fcst} = FCST ] ; then - if [[ $fn_tmpl =~ FHR3 ]] ; then - fhrs=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR3 - elif [[ ${fn_tmpl} =~ FHR2 ]] ; then - fhrs=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR2 - else - print_err_msg_exit "\ - Forecast file name templates are expected to contain a template - string, either FHR2 or FHR3" - fi - for fhr in ${fhrs[@]}; do - fns_in_arcv+=(${fn_tmpl/$tmpl/$fhr}) - done - fi - - # Make sure all filenames variables are set. - if [ -z $fns_in_arcv ] ; then - print_err_msg_exit "\ - The script has not set \$fns_in_arcv properly" - fi - - if [ -z ${fns_on_disk:-} ] ; then - fns_on_disk=(${fns_in_arcv[@]}) - fi - # - #----------------------------------------------------------------------- - # - # Set the system directory (i.e. a directory on disk) in which the external - # model output files for the specified cycle date (cdate) may be located. - # Note that this will be used by the calling script only if the output - # files for the specified cdate actually exist at this location. Otherwise, - # the files will be searched for on the mass store (HPSS). - # - #----------------------------------------------------------------------- - # - if [ "${anl_or_fcst}" = "ANL" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_ICS}) - elif [ "${anl_or_fcst}" = "FCST" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_LBCS}) - fi - - # - #----------------------------------------------------------------------- - # - # Set parameters associated with the mass store (HPSS) for the specified - # cycle date (cdate). These consist of: - # - # 1) The type of the archive file (e.g. tar, zip, etc). - # 2) The name of the archive file. - # 3) The full path in HPSS to the archive file. - # 4) The relative directory in the archive file in which the model output - # files are located. - # - # Note that these will be used by the calling script only if the archive - # file for the specified cdate actually exists on HPSS. - # - #----------------------------------------------------------------------- - # - case "${extrn_mdl_name}" in - - "GSMGFS") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="gpfs_hps_nco_ops_com_gfs_prod_gfs.${cdate}." - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}anl" - arcvrel_dir="." - elif [ "${anl_or_fcst}" = "FCST" ]; then - arcv_fns="${arcv_fns}sigma" - arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" - fi - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - ;; - - "FV3GFS") - - if [ "${cdate_FV3LAM}" -lt "2019061200" ]; then - arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" - arcv_fns="" - elif [ "${cdate_FV3LAM}" -ge "2019061200" ] && \ - [ "${cdate_FV3LAM}" -lt "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." - elif [ "${cdate_FV3LAM}" -ge "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." - fi - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_nemsioa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsiob" - else - arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) - fi - fi - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - arcv_fns="${arcv_fns}gfs_pgrb2" - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_netcdfa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_ncb" - else - arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) - fi - fi - - fi - - arcv_fmt="tar" - - slash_atmos_or_null="" - if [ "${cdate_FV3LAM}" -ge "2021032100" ]; then - slash_atmos_or_null="/atmos" - fi - arcvrel_dir="./gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" - - is_array arcv_fns - if [ "$?" = "0" ]; then - suffix=".${arcv_fmt}" - arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) - prefix="${arcv_dir}/" - arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) - else - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - fi - ;; - - - "RAP") - # - # Note that this is GSL RAPX data, not operational NCEP RAP data. - # An option for the latter may be added in the future. - # - # The zip archive files for RAPX are named such that the forecast - # files for odd-numbered starting hours (e.g. 01, 03, ..., 23) are - # stored together with the forecast files for the corresponding - # preceding even numbered starting hours (e.g. 00, 02, ..., 22, - # respectively), in an archive file whose name contains only the - # even-numbered hour. Thus, in forming the name of the archive - # file, if the starting hour (hh) is odd, we reduce it by one to get - # the corresponding even-numbered hour and use that to form the - # archive file name. - # - # Convert hh to a decimal (i.e. base-10) number to ovoid octal - # interpretation in bash. - - hh_orig=$hh - hh=$((10#$hh)) - if [ $(($hh%2)) = 1 ]; then - hh=$((hh-1)) - fi - # Archive files use 2-digit forecast hour - hh=$( printf "%02d\n" $hh ) - - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - - # Reset hh to its original value - hh=${hh_orig} - ;; - - "HRRR") - # - # Note that this is GSL HRRRX data, not operational NCEP HRRR data. - # An option for the latter may be added in the future. - # - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - "NAM") - # - # 12-km CONUS 218 grid. - # - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.awphys.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="." - ;; - - *) - print_err_msg_exit "\ -Archive file information has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - ;; - - esac - # - # Depending on the experiment configuration, the above code may set - # arcv_fns and arcv_fps to either scalars or arrays. If they are not - # arrays, recast them as arrays because that is what is expected in - # the code below. - # - is_array arcv_fns || arcv_fns=( "${arcv_fns}" ) - is_array arcv_fps || arcv_fps=( "${arcv_fps}" ) - # - #----------------------------------------------------------------------- - # - # Use the eval function to set the output variables. Note that each - # of these is set only if the corresponding input variable specifying - # the name to use for the output variable is not empty. - # - #----------------------------------------------------------------------- - # - if [ ! -z "${varname_extrn_mdl_cdate}" ]; then - eval ${varname_extrn_mdl_cdate}="${cdate}" - fi - - if [ ! -z "${varname_extrn_mdl_lbc_spec_fhrs}" ]; then - lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" - eval ${varname_extrn_mdl_lbc_spec_fhrs}=${lbc_spec_fhrs_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_on_disk}" ]; then - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}" )")" - eval ${varname_extrn_mdl_fns_on_disk}=${fns_on_disk_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_in_arcv}" ]; then - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}" )")" - eval ${varname_extrn_mdl_fns_in_arcv}=${fns_in_arcv_str} - fi - - if [ ! -z "${varname_extrn_mdl_sysdir}" ]; then - eval ${varname_extrn_mdl_sysdir}="${sysdir}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fmt}" ]; then - eval ${varname_extrn_mdl_arcv_fmt}="${arcv_fmt}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fns}" ]; then - arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}" )")" - eval ${varname_extrn_mdl_arcv_fns}=${arcv_fns_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fps}" ]; then - arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" - eval ${varname_extrn_mdl_arcv_fps}=${arcv_fps_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcvrel_dir}" ]; then - eval ${varname_extrn_mdl_arcvrel_dir}="${arcvrel_dir}" - fi - # - #----------------------------------------------------------------------- - # - # Restore the shell options saved at the beginning of this script/function. - # - #----------------------------------------------------------------------- - # - { restore_shell_opts; } > /dev/null 2>&1 -} diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py old mode 100644 new mode 100755 index 48210c0d53..86cb87e07b --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # pylint: disable=logging-fstring-interpolation ''' This script helps users pull data from known data streams, including @@ -31,6 +32,8 @@ import shutil import subprocess import sys +from textwrap import dedent + import yaml @@ -43,7 +46,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join(output_path,file_path) + local_file_path = os.path.join(output_path, file_path.lstrip("/")) if not os.path.exists(local_file_path): logging.info(f'File does not exist: {local_file_path}') unavailable['hpss'] = source_paths @@ -126,46 +129,6 @@ def download_file(url): return True -def download_requested_files(cla, data_store, store_specs): - - ''' This function interacts with the "download" protocol in a - provided data store specs file to download a set of files requested - by the user. It calls download_file for each individual file that - should be downloaded. ''' - - base_urls = store_specs['url'] - base_urls = base_urls if isinstance(base_urls, list) else [base_urls] - - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] - target_path = fill_template(cla.output_path, - cla.cycle_date) - - logging.info(f'Downloaded files will be placed here: \n {target_path}') - orig_path = os.getcwd() - os.chdir(target_path) - unavailable = {} - for base_url in base_urls: - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - url = os.path.join(base_url, file_name) - url = fill_template(url, cla.cycle_date, fcst_hr) - downloaded = download_file(url) - if not downloaded: - - if unavailable.get(data_store) is None: - unavailable[data_store] = [] - unavailable[data_store].append(target_path) - os.chdir(orig_path) - # Returning here assumes that if the first file - # isn't found, none of the others will be. Don't - # waste time timing out on every requested file. - return unavailable - os.chdir(orig_path) - return unavailable - def fhr_list(args): ''' @@ -184,7 +147,7 @@ def fhr_list(args): Must ensure that the list contains integers. ''' - args = args if isinstance(args, list) else [args] + args = args if isinstance(args, list) else list(args) arg_len = len(args) if arg_len in (2, 3): args[1] += 1 @@ -271,7 +234,7 @@ def find_archive_files(paths, file_names, cycle_date): return '', 0 -def get_requested_files(cla, file_names, input_loc, method='disk'): +def get_requested_files(cla, file_templates, input_loc, method='disk'): ''' This function copies files from disk locations or downloads files from a url, depending on the option specified for @@ -282,12 +245,12 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): Arguments: - cla Namespace object containing command line arguments - file_names Dict of file names by file type and kind - input_loc A string containing a single data location, either a url - or disk path. - method Choice of disk or download to indicate protocol for - retrieval + cla Namespace object containing command line arguments + file_templates a list of file templates + input_loc A string containing a single data location, either a url + or disk path. + method Choice of disk or download to indicate protocol for + retrieval Returns unavailable a dict whose keys are "method" and whose values are a @@ -296,12 +259,10 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): unavailable = {} - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] + logging.info(f'Getting files named like {file_templates}') - file_names = file_names if isinstance(file_names, list) else \ - [file_names] + file_templates = file_templates if isinstance(file_templates, list) else \ + [file_templates] target_path = fill_template(cla.output_path, cla.cycle_date) @@ -310,8 +271,9 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): os.chdir(target_path) unavailable = {} for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - loc = os.path.join(input_loc, file_name) + for file_template in file_templates: + loc = os.path.join(input_loc, file_template) + logging.debug(f'Full file path: {loc}') loc = fill_template(loc, cla.cycle_date, fcst_hr) if method == 'disk': @@ -358,7 +320,7 @@ def hsi_single_file(file_path, mode='ls'): return file_path -def hpss_requested_files(cla, store_specs): +def hpss_requested_files(cla, file_names, store_specs): ''' This function interacts with the "hpss" protocol in a provided data store specs file to download a set of files requested @@ -392,64 +354,70 @@ def hpss_requested_files(cla, store_specs): f' {list(zip(archive_paths, archive_file_names))}') existing_archive, which_archive = find_archive_files(archive_paths, - archive_file_names, - cla.cycle_date, - ) + archive_file_names, + cla.cycle_date, + ) if not existing_archive: logging.warning('No archive files were found!') unavailable['archive'] = list(zip(archive_paths, archive_file_names)) return unavailable - # Use the found archive file path to get the necessary files - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] + logging.info(f'Files in archive are named: {file_names}') - logging.debug(f'Grabbing archive number {which_archive} in list.') - archive_internal_dir = store_specs.get('archive_internal_dir', [''])[which_archive] - archive_internal_dir = fill_template(archive_internal_dir, - cla.cycle_date) + archive_internal_dirs = store_specs.get('archive_internal_dir', ['']) + if isinstance(archive_internal_dirs, dict): + archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, ['']) - output_path = fill_template(cla.output_path, cla.cycle_date) - logging.info(f'Will place files in {os.path.abspath(output_path)}') - orig_path = os.getcwd() - os.chdir(output_path) - logging.debug(f'CWD: {os.getcwd()}') + # which_archive matters for choosing the correct file names within, + # but we can safely just try all options for the + # archive_internal_dir + logging.debug(f'Checking archive number {which_archive} in list.') - source_paths = [] - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - source_paths.append(fill_template( - os.path.join(archive_internal_dir, file_name), - cla.cycle_date, - fcst_hr, - )) + for archive_internal_dir_tmpl in archive_internal_dirs: + archive_internal_dir = fill_template(archive_internal_dir_tmpl, + cla.cycle_date) - if store_specs.get('archive_format', 'tar') == 'zip': - # Get the entire file from HPSS - existing_archive = hsi_single_file(existing_archive, mode='get') + output_path = fill_template(cla.output_path, cla.cycle_date) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + orig_path = os.getcwd() + os.chdir(output_path) + logging.debug(f'CWD: {os.getcwd()}') - # Grab only the necessary files from the archive - cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + source_paths = [] + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + source_paths.append(fill_template( + os.path.join(archive_internal_dir, file_name), + cla.cycle_date, + fcst_hr, + )) - else: - cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + if store_specs.get('archive_format', 'tar') == 'zip': + # Get the entire file from HPSS + existing_archive = hsi_single_file(existing_archive, mode='get') - logging.info(f'Running command \n {cmd}') - subprocess.run(cmd, - check=True, - shell=True, - ) - - # Check that files exist and Remove any data transfer artifacts. - unavailable = clean_up_output_dir( - expected_subdir=archive_internal_dir, - local_archive=os.path.basename(existing_archive), - output_path=output_path, - source_paths=source_paths, - ) + # Grab only the necessary files from the archive + cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + + else: + cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) + + # Check that files exist and Remove any data transfer artifacts. + unavailable = clean_up_output_dir( + expected_subdir=archive_internal_dir, + local_archive=os.path.basename(existing_archive), + output_path=output_path, + source_paths=source_paths, + ) + if not unavailable: + return unavailable os.chdir(orig_path) @@ -505,6 +473,30 @@ def setup_logging(debug=False): +def write_summary_file(cla, data_store, file_templates): + + ''' Given the command line arguments and the data store from which the data + was retrieved, write a bash summary file that is needed by the workflow + elements downstream. ''' + + files = [] + for tmpl in file_templates: + files.extend([fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs]) + + summary_fp = os.path.join(cla.output_path, cla.summary_file) + logging.info(f'Writing a summary file to {summary_fp}') + file_contents = dedent(f''' + DATA_SRC={data_store} + EXTRN_MDL_CDATE={cla.cycle_date.strftime('%Y%m%d%H')} + EXTRN_MDL_STAGING_DIR={cla.output_path} + EXTRN_MDL_FNS=( {' '.join(files)} ) + EXTRN_MDL_FHRS=( {' '.join([str(i) for i in cla.fcst_hrs])} ) + ''') + logging.info(f'Contents: {file_contents}') + with open(summary_fp, "w") as summary: + summary.write(file_contents) + + def to_datetime(arg): ''' Return a datetime object give a string like YYYYMMDDHH. ''' @@ -521,29 +513,35 @@ def main(cla): paths in priority order. ''' - setup_logging(cla.debug) - - known_data_info = cla.config.get(cla.external_model) - if known_data_info is None: - msg = ('No data stores have been defined for', - f'{cla.external_model}!') - raise KeyError(msg) + data_stores = cla.data_stores + known_data_info = cla.config.get(cla.external_model, {}) + if not known_data_info: + msg = dedent(f'''No data stores have been defined for + {cla.external_model}!''') + if cla.input_file_path is None: + data_stores = ['disk'] + raise KeyError(msg) + logging.info(msg + ' Only checking provided disk location.') unavailable = {} - for data_store in cla.data_stores: + for data_store in data_stores: logging.info(f'Checking {data_store} for {cla.external_model}') store_specs = known_data_info.get(data_store, {}) if data_store == 'disk': - file_names = cla.file_names if cla.file_names else \ + file_templates = cla.file_templates if cla.file_templates else \ known_data_info.get('hpss', {}).get('file_names') - logging.debug(f'User supplied file names are: {file_names}') - if not file_names: - msg = ('No file name found. They must be provided \ + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + logging.debug(f'User supplied file names are: {file_templates}') + if not file_templates: + msg = ('No file naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) unavailable = get_requested_files(cla, - file_names=file_names, + file_templates=file_templates, input_loc=cla.input_file_path, method='disk', ) @@ -552,24 +550,32 @@ def main(cla): msg = (f'No information is available for {data_store}.') raise KeyError(msg) - if store_specs.get('protocol') == 'download': - file_names = store_specs.get('file_names') - if not file_names: - msg = ('No file name found. They must be provided \ + else: + + file_templates = store_specs.get('file_names') + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + if not file_templates: + msg = ('No file name naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) - unavailable = get_requested_files(cla, - file_names=file_names, - input_loc=store_specs['url'], - method='download', - ) - - if store_specs.get('protocol') == 'htar': - unavailable = hpss_requested_files(cla, store_specs) + if store_specs.get('protocol') == 'download': + unavailable = get_requested_files(cla, + file_templates=file_templates, + input_loc=store_specs['url'], + method='download', + ) + if store_specs.get('protocol') == 'htar': + unavailable = hpss_requested_files(cla, file_templates, store_specs) if not unavailable: # All files are found. Stop looking! + # Write a variable definitions file for the data, if requested + if cla.summary_file: + write_summary_file(cla, data_store, file_templates) break logging.warning(f'Requested files are unavialable from {data_store}') @@ -657,11 +663,11 @@ def parse_args(): help='Print debug messages', ) parser.add_argument( - '--file_names', - help='A YAML-formatted string that indicates the naming \ + '--file_templates', + help='One or more file template strings defining the naming \ convention the be used for the files retrieved from disk. If \ not provided, the default names from hpss are used.', - type=load_str, + nargs='*', ) parser.add_argument( '--file_type', @@ -672,9 +678,13 @@ def parse_args(): '--input_file_path', help='A path to data stored on disk. The path may contain \ Python templates. File names may be supplied using the \ - --file_names flag, or the default naming convention will be \ + --file_templates flag, or the default naming convention will be \ taken from the --config file.', - nargs='*', + ) + parser.add_argument( + '--summary_file', + help='Name of the summary file to be written to the output \ + directory', ) return parser.parse_args() @@ -684,6 +694,15 @@ def parse_args(): CLA.output_path = path_exists(CLA.output_path) CLA.fcst_hrs = fhr_list(CLA.fcst_hrs) + + setup_logging(CLA.debug) + print(f"Running script retrieve_data.py with args:\n", + f"{('-' * 80)}\n{('-' * 80)}") + for name, val in CLA.__dict__.items(): + if name not in ['config']: + print(f"{name:>15s}: {val}") + print(f"{('-' * 80)}\n{('-' * 80)}") + if 'disk' in CLA.data_stores: # Make sure a path was provided. if not CLA.input_file_path: diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 48e53d81a6..6cdaff3474 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -278,7 +278,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_ics }} ICS_OR_LBCSICS @@ -305,7 +304,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_lbcs }} ICS_OR_LBCSLBCS diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index c3efd9a29d..d82eeec677 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -65,6 +65,13 @@ FV3GFS: fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio + netcdf: + anl: + - gfs.t{hh}z.atmanl.nc + - gfs.t{hh}z.sfcanl.nc + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nc + - gfs.t{hh}z.sfcf{fcst_hr:03d}.nc hpss: protocol: htar archive_path: @@ -103,16 +110,41 @@ FV3GFS: file_names: <<: *gfs_file_names +GSMGFS: + hpss: + protocol: htar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + anl: + - ./ + fcst: + - /gpfs/hps/nco/ops/com/gfs/prod/gfs.{yyyymmdd} + archive_file_names: + anl: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.anl.tar + fcst: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.sigma.tar + file_names: + anl: + - gfs.t{hh}z.atmanl.nemsio + - gfs.t{hh}z.sfcanl.nemsio + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio + RAP: hpss: protocol: htar archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # RAP forecasts are binned into 6 hour tar files. + - gpfs_hps_nco_ops_com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar - com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar file_names: &rap_file_names anl: @@ -125,58 +157,31 @@ RAP: file_names: <<: *rap_file_names -RAPx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/rap/full/wrfnat - archive_file_names: - # RAPx bins two cycles togehter, and named by the lower even value - # of the cycle hour. - - '{yyyymmdd}{hh_even}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - HRRR: hpss: protocol: htar archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # HRRR forecasts are binned into 6 hour tar files. - - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrfnatdng.tar + - gpfs_hps_nco_ops_com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar + - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar file_names: &hrrr_file_names anl: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 fcst: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 aws: protocol: download url: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.{yyyymmdd}/conus/ file_names: <<: *hrrr_file_names -HRRRx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/hrrr/conus/wrfnat - archive_file_names: - - '{yyyymmddhh}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - NAM: hpss: protocol: htar From 09025104c905c8c69c36363aa70a551c1d2e6aa9 Mon Sep 17 00:00:00 2001 From: BenjaminBlake-NOAA <52074832+BenjaminBlake-NOAA@users.noreply.github.com> Date: Tue, 12 Apr 2022 21:08:14 -0400 Subject: [PATCH 151/203] Increase size of RRFS CONUS grid (#724) Co-authored-by: Benjamin.Blake EMC Co-authored-by: Benjamin.Blake EMC Co-authored-by: Benjamin.Blake EMC Co-authored-by: chan-hoo --- tests/WE2E/machine_suites/hera.txt | 12 +- tests/WE2E/machine_suites/jet.txt | 12 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 25 ++++ ...pact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} | 4 +- ...m_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} | 7 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} | 7 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 25 ++++ ...act_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh} | 4 +- ...m_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh} | 4 +- ...pact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} | 7 +- ...m_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} | 7 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} | 7 +- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 25 ++++ ..._3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh} | 7 +- ...mpact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} | 7 +- ...m_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} | 8 +- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} | 8 +- ...pact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} | 6 +- .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 2 +- .../wflow_features/config.nco_inline_post.sh | 2 +- ...ig.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh | 2 +- .../wflow_features/config.subhourly_post.sh | 2 +- .../config.subhourly_post_ensemble_2mems.sh | 2 +- ush/set_predef_grid_params.sh | 138 ++++++++++++++++++ ush/valid_param_vals.sh | 3 + 25 files changed, 271 insertions(+), 62 deletions(-) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh => config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} (80%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh => config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} (77%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh => config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} (77%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh => config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh} (80%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh => config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh} (79%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh => config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} (78%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh => config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} (77%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh => config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} (77%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh => config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh} (78%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh => config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} (78%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh => config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh} (67%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh => config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh} (67%) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/{config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh => config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh} (70%) diff --git a/tests/WE2E/machine_suites/hera.txt b/tests/WE2E/machine_suites/hera.txt index 9316d68898..a3f7f28509 100644 --- a/tests/WE2E/machine_suites/hera.txt +++ b/tests/WE2E/machine_suites/hera.txt @@ -2,9 +2,9 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/machine_suites/jet.txt b/tests/WE2E/machine_suites/jet.txt index 9316d68898..a3f7f28509 100644 --- a/tests/WE2E/machine_suites/jet.txt +++ b/tests/WE2E/machine_suites/jet.txt @@ -2,9 +2,9 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha -grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha +grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta +nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..d7c402004c --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUScompact_13km grid using the +# GFS_v16 physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUScompact_13km" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 80% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index bec80f24aa..4239752de4 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -3,7 +3,7 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_13km grid using the HRRR +# completes successfully on the RRFS_CONUScompact_13km grid using the HRRR # physics suite with ICs derived from the HRRR and LBCs derived from the # RAP. # @@ -11,7 +11,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_13km" +PREDEF_GRID_NAME="RRFS_CONUScompact_13km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 77% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 5637d0b7a4..8aaeddf43c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_3km grid using the RRFS_v1alpha -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_13km grid using the RRFS_v1alpha +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_3km" +PREDEF_GRID_NAME="RRFS_CONUScompact_13km" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 77% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 07f66f9ccc..9e86198e6f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_3km grid using the RRFS_v1beta -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_13km grid using the RRFS_v1beta +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_3km" +PREDEF_GRID_NAME="RRFS_CONUScompact_13km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..f5fc6384bf --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUScompact_25km grid using the +# GFS_v16 physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh similarity index 80% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh index 0e8ad1d04a..67b381044c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -3,14 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# completes successfully on the RRFS_CONUScompact_25km grid using the HRRR # physics suite with ICs and LBCs derived from the HRRR. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh similarity index 79% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh index f236d2f35e..09b9548d11 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -3,14 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta +# completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta # physics suite with ICs and LBCs derived from the HRRR. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 78% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 01471df077..f4b781e070 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the HRRR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_25km grid using the HRRR +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 77% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 48f2f4c13c..b0b8a2e42d 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1alpha -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1alpha +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 77% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index f5fabd9811..2a98569a39 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh new file mode 100644 index 0000000000..5cd9b09042 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUScompact_3km grid using the +# GFS_v16 physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUScompact_3km" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh similarity index 78% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 960f4b1860..2407157518 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_3km grid using the GFS_v15p2 -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_3km grid using the GFS_v15p2 +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_3km" +PREDEF_GRID_NAME="RRFS_CONUScompact_3km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 78% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 56b1ffb8be..eda52d1e7e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -3,15 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_3km grid using the HRRR -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_3km grid using the HRRR +# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_3km" +PREDEF_GRID_NAME="RRFS_CONUScompact_3km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh similarity index 67% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index b3b420e003..1b3131663d 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -3,15 +3,15 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_13km grid using the RRFS_v1alpha -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_3km grid using the +# RRFS_v1alpha physics suite with ICs derived from the HRRR and LBCs +# derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_13km" +PREDEF_GRID_NAME="RRFS_CONUScompact_3km" CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh similarity index 67% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 3f39ab0f72..cf6965435f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -3,15 +3,15 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUS_13km grid using the RRFS_v1beta -# physics suite with ICs derived from the HRRR and LBCs derived from the -# RAP. +# completes successfully on the RRFS_CONUScompact_3km grid using the +# RRFS_v1beta physics suite with ICs derived from the HRRR and LBCs +# derived from the RAP. # RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_13km" +PREDEF_GRID_NAME="RRFS_CONUScompact_3km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh similarity index 70% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 0a2d426fa2..c2a054b309 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -3,14 +3,14 @@ # ------------------------ # # This test is to ensure that the workflow running in nco mode completes -# successfully on the RRFS_CONUS_25km grid using the HRRR physics suite -# with ICs derived from the HRRR and LBCs derived from the RAP. +# successfully on the RRFS_CONUScompact_25km grid using the HRRR physics +# suite with ICs derived from the HRRR and LBCs derived from the RAP. # RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh index d535bc5d54..5d6ed126ec 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh @@ -12,7 +12,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh index 35901eeb53..392d3311ac 120000 --- a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh @@ -1 +1 @@ -../grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh \ No newline at end of file +../grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh \ No newline at end of file diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index 18699dd51d..d985559f2e 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -10,7 +10,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh index 3edce24dff..09df72e29a 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh @@ -9,7 +9,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh index 337e997401..4c20807119 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh @@ -15,7 +15,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="RRFS_CONUS_25km" +PREDEF_GRID_NAME="RRFS_CONUScompact_25km" CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 53aaf7c603..88b2c0f633 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -109,6 +109,52 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="25000.0" ESGgrid_DELY="25000.0" + ESGgrid_NX="219" + ESGgrid_NY="131" + + ESGgrid_PAZI="0.0" + + ESGgrid_WIDE_HALO_WIDTH="6" + + DT_ATMOS="${DT_ATMOS:-40}" + + LAYOUT_X="${LAYOUT_X:-5}" + LAYOUT_Y="${LAYOUT_Y:-2}" + BLOCKSIZE="${BLOCKSIZE:-40}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="2" + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="217" + WRTCMP_ny="128" + WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lat_lwr_left="21.138123" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" + fi + ;; +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~25km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_25km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" + + ESGgrid_DELX="25000.0" + ESGgrid_DELY="25000.0" + ESGgrid_NX="202" ESGgrid_NY="116" @@ -155,6 +201,52 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="13000.0" ESGgrid_DELY="13000.0" + ESGgrid_NX="420" + ESGgrid_NY="252" + + ESGgrid_PAZI="0.0" + + ESGgrid_WIDE_HALO_WIDTH="6" + + DT_ATMOS="${DT_ATMOS:-45}" + + LAYOUT_X="${LAYOUT_X:-16}" + LAYOUT_Y="${LAYOUT_Y:-10}" + BLOCKSIZE="${BLOCKSIZE:-32}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="416" + WRTCMP_ny="245" + WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lat_lwr_left="21.138123" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" + fi + ;; +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~13km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_13km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" + + ESGgrid_DELX="13000.0" + ESGgrid_DELY="13000.0" + ESGgrid_NX="396" ESGgrid_NY="232" @@ -201,6 +293,52 @@ case ${PREDEF_GRID_NAME} in ESGgrid_DELX="3000.0" ESGgrid_DELY="3000.0" + ESGgrid_NX="1820" + ESGgrid_NY="1092" + + ESGgrid_PAZI="0.0" + + ESGgrid_WIDE_HALO_WIDTH="6" + + DT_ATMOS="${DT_ATMOS:-36}" + + LAYOUT_X="${LAYOUT_X:-28}" + LAYOUT_Y="${LAYOUT_Y:-28}" + BLOCKSIZE="${BLOCKSIZE:-29}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="1799" + WRTCMP_ny="1059" + WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lat_lwr_left="21.138123" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" + fi + ;; +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~3km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_3km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" + + ESGgrid_DELX="3000.0" + ESGgrid_DELY="3000.0" + ESGgrid_NX="1748" ESGgrid_NY="1038" diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a41f8eb86f..d7fdd9f071 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -12,6 +12,9 @@ valid_vals_PREDEF_GRID_NAME=( \ "RRFS_CONUS_25km" \ "RRFS_CONUS_13km" \ "RRFS_CONUS_3km" \ +"RRFS_CONUScompact_25km" \ +"RRFS_CONUScompact_13km" \ +"RRFS_CONUScompact_3km" \ "RRFS_SUBCONUS_3km" \ "RRFS_AK_13km" \ "RRFS_AK_3km" \ From bc50915f6df97f8557cde50dc133eb85481505a3 Mon Sep 17 00:00:00 2001 From: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Date: Sun, 17 Apr 2022 13:54:43 -0600 Subject: [PATCH 152/203] add include-style quality mark options in metplus confs (#738) --- ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf | 3 +++ ush/templates/parm/metplus/EnsembleStat_upper_air.conf | 3 +++ ush/templates/parm/metplus/PointStat_conus_sfc.conf | 3 +-- ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf | 3 +-- ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf | 3 +-- ush/templates/parm/metplus/PointStat_upper_air.conf | 3 +-- ush/templates/parm/metplus/PointStat_upper_air_mean.conf | 3 +-- ush/templates/parm/metplus/PointStat_upper_air_prob.conf | 3 +-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index ecdc7f1ac9..e1d95040c0 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -93,6 +93,9 @@ ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped +ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +#ENSEMBLE_STAT_OBS_QUALITY_EXC = + # if True, pb2nc will skip processing a file if the output already exists # used to speed up runs and reduce redundancy PB2NC_SKIP_IF_OUTPUT_EXISTS = True diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf index d25241e936..6251fbac13 100644 --- a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -93,6 +93,9 @@ ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped +ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +#ENSEMBLE_STAT_OBS_QUALITY_EXC = + # if True, pb2nc will skip processing a file if the output already exists # used to speed up runs and reduce redundancy PB2NC_SKIP_IF_OUTPUT_EXISTS = True diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 1558c4aa99..f6e231699c 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf index db3b3f9329..20e2d7392a 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf index a1754261e7..f28d022441 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index e21f2566c0..1f24cbab46 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf index 5152536831..a2b655136c 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf index 0a15f6f1f0..0212f79d42 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -91,8 +91,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped - -#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST From 6701c61da184d7631d2240dc4030a3f3e4091cd0 Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Thu, 21 Apr 2022 21:16:33 -0400 Subject: [PATCH 153/203] Add Gaea as a supported platform for the regional_workflow (#734) * Updates to port regional workflow to gaea * Temp change with -v as batch option * new fixes for gaea/slurm * Updated time for make lbcs * added TEST data directory path * Update gaea.sh * fixes for PR --- modulefiles/tasks/gaea/make_grid.local | 6 ++ modulefiles/tasks/gaea/make_ics.local | 6 ++ modulefiles/tasks/gaea/make_lbcs.local | 6 ++ modulefiles/tasks/gaea/run_fcst.local | 6 ++ modulefiles/tasks/gaea/run_vx.local | 7 ++ ush/config_defaults.sh | 8 ++ ush/generate_FV3LAM_wflow.sh | 1 + ush/machine/gaea.sh | 70 ++++++++++++++ ush/templates/FV3LAM_wflow.xml | 124 +++++++++++++++++++++++++ ush/valid_param_vals.sh | 2 +- 10 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 modulefiles/tasks/gaea/make_grid.local create mode 100644 modulefiles/tasks/gaea/make_ics.local create mode 100644 modulefiles/tasks/gaea/make_lbcs.local create mode 100644 modulefiles/tasks/gaea/run_fcst.local create mode 100644 modulefiles/tasks/gaea/run_vx.local create mode 100755 ush/machine/gaea.sh diff --git a/modulefiles/tasks/gaea/make_grid.local b/modulefiles/tasks/gaea/make_grid.local new file mode 100644 index 0000000000..fbd52ffa08 --- /dev/null +++ b/modulefiles/tasks/gaea/make_grid.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/make_ics.local b/modulefiles/tasks/gaea/make_ics.local new file mode 100644 index 0000000000..fbd52ffa08 --- /dev/null +++ b/modulefiles/tasks/gaea/make_ics.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/make_lbcs.local b/modulefiles/tasks/gaea/make_lbcs.local new file mode 100644 index 0000000000..fbd52ffa08 --- /dev/null +++ b/modulefiles/tasks/gaea/make_lbcs.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/run_fcst.local b/modulefiles/tasks/gaea/run_fcst.local new file mode 100644 index 0000000000..fbd52ffa08 --- /dev/null +++ b/modulefiles/tasks/gaea/run_fcst.local @@ -0,0 +1,6 @@ +#%Module +module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles +module load rocoto +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/gaea/run_vx.local b/modulefiles/tasks/gaea/run_vx.local new file mode 100644 index 0000000000..929c2011f8 --- /dev/null +++ b/modulefiles/tasks/gaea/run_vx.local @@ -0,0 +1,7 @@ +#%Module + +module use -a /contrib/anaconda/modulefiles +module load intel/18.0.5.274 +module load anaconda/latest +module use -a /contrib/met/modulefiles/ +module load met/10.0.0 diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 145d397ff3..9345704876 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1691,6 +1691,14 @@ MAXTRIES_VX_ENSGRID_PROB_RETOP="1" MAXTRIES_VX_ENSPOINT="1" MAXTRIES_VX_ENSPOINT_MEAN="1" MAXTRIES_VX_ENSPOINT_PROB="1" + +# +#----------------------------------------------------------------------- +# +# Allows an extra parameter to be passed to slurm via XML Native +# command +# +SLURM_NATIVE_CMD="" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 83282b7177..8af7497b03 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -193,6 +193,7 @@ file (template_xml_fp): 'partition_fcst': ${PARTITION_FCST} 'queue_fcst': ${QUEUE_FCST} 'machine': ${MACHINE} + 'slurm_native_cmd': ${SLURM_NATIVE_CMD} # # Workflow task names. # diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh new file mode 100755 index 0000000000..9bad0af743 --- /dev/null +++ b/ush/machine/gaea.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data/FV3GFS/${yyyymmdd}${hh}' + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + +# System scripts to source to initialize various commands within workflow +# scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="rocoto" +SLURM_NATIVE_CMD="-M c3" +NCORES_PER_NODE=${NCORES_PER_NODE:-32} +SCHED=${SCHED:-"slurm"} +QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} +QUEUE_HPSS=${QUEUE_DEFAULT:-"normal"} +QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} +WTIME_MAKE_LBCS="00:60:00" + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"//lustre/f2/dev/Mark.Potts/EPIC/fix/fix_am"} +FIXaer=${FIXaer:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_aer"} +FIXlut=${FIXlut:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_sfc_climo"} + +RUN_CMD_SERIAL="time" +#Run Commands currently differ for GNU/openmpi +#RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' +#RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' +#RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' +RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' +RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' +RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' + +# MET Installation Locations +# MET Plus is not yet supported on gaea +# Test Data Locations +TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" + diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 6cdaff3474..8dff08780d 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -22,6 +22,7 @@ Parameters needed by the job scheduler. + @@ -184,6 +185,9 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &MAKE_GRID_TN; &LOGDIR;/&MAKE_GRID_TN;.log @@ -207,6 +211,9 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &MAKE_OROG_TN; &LOGDIR;/&MAKE_OROG_TN;.log @@ -234,6 +241,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_sfc_climo }}:ppn={{ ppn_make_sfc_climo }} {{ wtime_make_sfc_climo }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &MAKE_SFC_CLIMO_TN; &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log @@ -271,6 +281,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &GET_EXTRN_ICS_TN; &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log @@ -297,6 +310,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &GET_EXTRN_LBCS_TN; &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log @@ -330,6 +346,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} {{ wtime_make_ics }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &MAKE_ICS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -374,6 +393,9 @@ MODULES_RUN_TASK_FP script. {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} {{ wtime_make_lbcs }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &MAKE_LBCS_TN;{{ uscore_ensmem_name }} &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -421,6 +443,9 @@ MODULES_RUN_TASK_FP script. {%- else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; + {%- endif %} + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; {%- endif %} {{ wtime_run_fcst }} &RUN_FCST_TN;{{ uscore_ensmem_name }} @@ -473,6 +498,9 @@ later below for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -527,6 +555,9 @@ for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -586,6 +617,9 @@ always zero). {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} {%- if sub_hourly_post %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -654,6 +688,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log @@ -697,6 +734,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ccpa }}:ppn={{ ppn_get_obs_ccpa }} {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &GET_OBS_CCPA_TN; &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log @@ -727,6 +767,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_mrms }}:ppn={{ ppn_get_obs_mrms }} {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &GET_OBS_MRMS_TN; &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log @@ -758,6 +801,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_get_obs_ndas }}:ppn={{ ppn_get_obs_ndas }} {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &GET_OBS_NDAS_TN; &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log @@ -784,6 +830,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_TN; &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -834,6 +883,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_REFC_TN; &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -883,6 +935,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_RETOP_TN; &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -932,6 +987,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_03h_TN; &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -967,6 +1025,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_06h_TN; &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1002,6 +1063,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_gridstat }}:ppn={{ ppn_vx_gridstat }} {{ wtime_vx_gridstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_GRIDSTAT_24h_TN; &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1037,6 +1101,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_pointstat }}:ppn={{ ppn_vx_pointstat }} {{ wtime_vx_pointstat }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_POINTSTAT_TN; &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1089,6 +1156,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_TN; &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log @@ -1121,6 +1191,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_03h_TN; &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log @@ -1153,6 +1226,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_06h_TN; &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log @@ -1185,6 +1261,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_24h_TN; &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log @@ -1216,6 +1295,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_REFC_TN; &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log @@ -1246,6 +1328,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid }}:ppn={{ ppn_vx_ensgrid }} {{ wtime_vx_ensgrid }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_RETOP_TN; &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log @@ -1275,6 +1360,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_MEAN_TN; &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log @@ -1306,6 +1394,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_TN; &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log @@ -1337,6 +1428,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_MEAN_03h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log @@ -1368,6 +1462,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_03h_TN; &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log @@ -1400,6 +1497,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_MEAN_06h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log @@ -1431,6 +1531,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_06h_TN; &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log @@ -1463,6 +1566,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_mean }}:ppn={{ ppn_vx_ensgrid_mean }} {{ wtime_vx_ensgrid_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_MEAN_24h_TN; &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log @@ -1494,6 +1600,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_24h_TN; &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log @@ -1525,6 +1634,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_REFC_TN; &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log @@ -1555,6 +1667,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_ensgrid_prob }}:ppn={{ ppn_vx_ensgrid_prob }} {{ wtime_vx_ensgrid_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSGRID_PROB_RETOP_TN; &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log @@ -1586,6 +1701,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint }}:ppn={{ ppn_vx_enspoint }} {{ wtime_vx_enspoint }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSPOINT_TN; &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log @@ -1614,6 +1732,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_mean }}:ppn={{ ppn_vx_enspoint_mean }} {{ wtime_vx_enspoint_mean }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSPOINT_MEAN_TN; &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log @@ -1642,6 +1763,9 @@ the tag to be identical to the ones above for other output times. {{ nnodes_vx_enspoint_prob }}:ppn={{ ppn_vx_enspoint_prob }} {{ wtime_vx_enspoint_prob }} &NCORES_PER_NODE; + {%- if machine in ["GAEA"] %} + &SLURM_NATIVE_CMD; + {%- endif %} &VX_ENSPOINT_PROB_TN; &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index d7fdd9f071..a22e51d540 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY") +valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY" "GAEA") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") valid_vals_WORKFLOW_MANAGER=("rocoto" "none") From 6509e644e4c777b3a6209efe596b4f0a875e4094 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Fri, 22 Apr 2022 11:18:48 -0600 Subject: [PATCH 154/203] Add more parameters to CSV file containing WE2E test info (#740) ## DESCRIPTION OF CHANGES: The script/function `get_WE2Etest_names_subdirs_descs.sh` (which is called from `run_WE2E_tests.sh` if needed) creates a CSV (Comma-Separated Value) file named `WE2E_test_info.csv` that contains information about the WE2E tests. Currently, this CSV file contains only 3 columns: the test name, the names of any alternate names for the test, and the test description. In order to have a more complete summary of the WE2E tests, this PR modifies `get_WE2Etest_names_subdirs_descs.sh` so that additional information is included in the CSV file. This additional information consists of the values of the following experiment variables for each test: ``` PREDEF_GRID_NAME CCPP_PHYS_SUITE EXTRN_MDL_NAME_ICS EXTRN_MDL_NAME_LBCS DATE_FIRST_CYCL DATE_LAST_CYCL CYCL_HRS INCR_CYCL_FREQ FCST_LEN_HRS LBC_SPEC_INTVL_HRS NUM_ENS_MEMBERS ``` In addition, the script uses this information to calculate the number of times each test calls the forecast model (e.g. if the test uses 3 different cycle dates, then the forecast model will be called 3 times; if it is an ensemble test for a single cycle, the test will call the forecast model as many times as the number of ensemble members). ## TESTS CONDUCTED: The script `run_WE2E_tests.sh` was called that in turn calls `get_WE2Etest_names_subdirs_descs.sh`. This created a new CSV file that contained the new fields (columns). The CSV file was imported into Google Sheets (using "|" as the field/column separator) and looked correct. ## DOCUMENTATION: The documentation is for the most part already within the `get_WE2Etest_names_subdirs_descs.sh`. This PR slightly modifies that documentation to update it. --- scripts/exregional_make_grid.sh | 2 +- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 306 ++++++++++++++++-- ush/check_expt_config_vars.sh | 2 +- ush/setup.sh | 3 +- 4 files changed, 288 insertions(+), 25 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 742770630b..3744638c8d 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -410,7 +410,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then CRES="C${res_equiv}" fi -set_file_param "${GLOBAL_VAR_DEFNS_FP}" "CRES" "\"$CRES\"" +set_file_param "${GLOBAL_VAR_DEFNS_FP}" "CRES" "'$CRES'" # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 2183ead891..d0db858b33 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -7,8 +7,10 @@ # the WE2E tests available in the WE2E testing system. This information # consists of the test names, the category subdirectories in which the # test configuration files are located (relative to a base directory), -# the test IDs, and the test descriptions. These are described in more -# detail below. +# the test IDs, and the test descriptions. This function optionally +# also creates a CSV (Comma-Separated Value) file containing various +# pieces of information about each of the workflow end-to-end (WE2E) +# tests. These are described in more detail below. # # The function takes as inputs the following arguments: # @@ -21,6 +23,10 @@ # Flag that specifies whether or not a CSV (Comma-Separated Value) file # containing information about the WE2E tests should be generated. # +# verbose: +# Optional verbosity flag. Should be set to "TRUE" or "FALSE". Default +# is "FALSE". +# # output_varname_test_configs_basedir: # Name of output variable in which to return the base directory of the # WE2E test configuration files. @@ -65,7 +71,7 @@ # in the local array category_subdirs. We refer to these as "category" # subdirectories because they are used for clarity to group the tests # into categories (instead of putting them all directly under the base -# directory). For example, one category of tests might be those that +# directory). For example, one category of tests might be those that # test workflow capabilities such as running multiple cycles and ensemble # forecasts, another might be those that run various combinations of # grids, physics suites, and external models for ICs/LBCs, etc. Note @@ -208,7 +214,11 @@ # the file will be placed in the main WE2E testing system directory # specified by the input argument WE2Edir. The CSV file can be read # into a spreadsheet in Google Sheets (or another similar tool) to get -# an overview of all the available WE2E tests. +# an overview of all the available WE2E tests. The rows of the CSV file +# correspond to the primary WE2E tests, and the columns correspond to +# the (primary) test name, alternate test names (if any), test description, +# number of times the test calls the forecast model, and values of various +# SRW App experiment variables for that test. # # A CSV file will be generated in the directory specified by WE2Edir if # one or more of the following conditions hold: @@ -228,7 +238,8 @@ # "FALSE" in the call to this function (regardless of whether or not a # CSV file already exists). If a CSV file is generated, it is placed in # the directory specified by the input argment WE2Edir, and it overwrites -# any existing copies of the file in that directory. +# any existing copies of the file in that directory. The contents of +# each column of the CSV file are described below. # #----------------------------------------------------------------------- # @@ -255,6 +266,7 @@ function get_WE2Etest_names_subdirs_descs() { local valid_args=( \ "WE2Edir" \ "generate_csv_file" \ + "verbose" \ "output_varname_test_configs_basedir" \ "output_varname_test_names" \ "output_varname_test_subdirs" \ @@ -275,6 +287,17 @@ function get_WE2Etest_names_subdirs_descs() { # #----------------------------------------------------------------------- # +# Make the default value of "verbose" "FALSE". Then make sure "verbose" +# is set to a valid value. +# +#----------------------------------------------------------------------- +# + verbose=${verbose:-"FALSE"} + check_var_valid_value "verbose" "valid_vals_BOOLEAN" + verbose=$(boolify $verbose) +# +#----------------------------------------------------------------------- +# # Declare local variables. # #----------------------------------------------------------------------- @@ -286,7 +309,10 @@ function get_WE2Etest_names_subdirs_descs() { alt_test_prim_test_names \ alt_test_subdir \ alt_test_subdirs \ + array_names_vars_to_extract \ + array_names_vars_to_extract_orig \ category_subdirs \ + cmd \ column_titles \ config_fn \ crnt_item \ @@ -294,24 +320,35 @@ function get_WE2Etest_names_subdirs_descs() { csv_fn \ csv_fp \ cwd \ + default_val \ hash_or_null \ i \ ii \ j \ jp1 \ + k \ line \ mod_time_csv \ mod_time_subdir \ + msg \ num_alt_tests \ num_category_subdirs \ + num_cdates \ + num_cycles_per_day \ + num_days \ + num_fcsts \ + num_fcsts_orig \ num_items \ num_occurrences \ num_prim_tests \ num_tests \ + num_vars_to_extract \ + prim_array_names_vars_to_extract \ prim_test_descs \ prim_test_ids \ prim_test_name_subdir \ prim_test_names \ + prim_test_num_fcsts \ prim_test_subdirs \ get_test_descs \ regex_search \ @@ -349,7 +386,11 @@ function get_WE2Etest_names_subdirs_descs() { test_subdirs_orig \ test_subdirs_str \ test_type \ - valid_vals_generate_csv_file + val \ + valid_vals_generate_csv_file \ + var_name \ + var_name_at \ + vars_to_extract # #----------------------------------------------------------------------- # @@ -408,6 +449,13 @@ function get_WE2Etest_names_subdirs_descs() { fi fi + + if [ "${generate_csv_file}" = "TRUE" ]; then + print_info_msg " +Will generate a CSV (Comma Separated Value) file (csv_fp) containing +information on all WE2E tests: + csv_fp = \"${csv_fp}\"" + fi # #----------------------------------------------------------------------- # @@ -472,6 +520,7 @@ function get_WE2Etest_names_subdirs_descs() { prim_test_names=() prim_test_ids=() prim_test_subdirs=() + prim_test_num_fcsts=() alt_test_names=() alt_test_subdirs=() @@ -839,17 +888,61 @@ they correspond to unique test names and rerun." #----------------------------------------------------------------------- # if [ "${get_test_descs}" = "TRUE" ]; then +# +# Specify in "vars_to_extract" the list of experiment variables to extract +# from each test configuration file (and later to place in the CSV file). +# Recall that the rows of the CSV file correspond to the various WE2E +# tests, and the columns correspond to the test name, description, and +# experiment variable values. The elements of "vars_to_extract" should +# be the names of SRW App experiment variables that are (or can be) +# specified in the App's configuration file. Note that if a variable is +# not specified in the test configuration file, in most cases its value +# is set to an empty string (and recorded as such in the CSV file). In +# some cases, it is set to some other value (e.g. for the number of +# ensemble members NUM_ENS_MEMBERS, it is set to 1). +# + vars_to_extract=( "PREDEF_GRID_NAME" \ + "CCPP_PHYS_SUITE" \ + "EXTRN_MDL_NAME_ICS" \ + "EXTRN_MDL_NAME_LBCS" \ + "DATE_FIRST_CYCL" \ + "DATE_LAST_CYCL" \ + "CYCL_HRS" \ + "INCR_CYCL_FREQ" \ + "FCST_LEN_HRS" \ + "LBC_SPEC_INTVL_HRS" \ + "NUM_ENS_MEMBERS" \ + ) + num_vars_to_extract="${#vars_to_extract[@]}" +# +# Create names of local arrays that will hold the value of the corresponding +# variable for each test. Then use these names to define them as empty +# arrays. [The arrays named "prim_..." are to hold values for only the +# primary tests, while other arrays are to hold values for all (primary +# plus alternate) tests.] +# + prim_array_names_vars_to_extract=( $( printf "prim_test_%s_vals " "${vars_to_extract[@]}" ) ) + array_names_vars_to_extract=( $( printf "%s_vals " "${vars_to_extract[@]}" ) ) + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + cmd="${prim_array_names_vars_to_extract[$k]}=()" + eval $cmd + cmd="${array_names_vars_to_extract[$k]}=()" + eval $cmd + done print_info_msg " -Gathering test descriptions from the configuration files of the primary -WE2E tests..." +Gathering test descriptions and experiment variable values from the +configuration files of the primary WE2E tests... +" prim_test_descs=() for (( i=0; i<=$((num_prim_tests-1)); i++ )); do test_name="${prim_test_names[$i]}" print_info_msg "\ - Reading in the test description for primary WE2E test: \"${test_name}\"" + Reading in the test description for primary WE2E test: \"${test_name}\" + In category (subdirectory): \"${subdir}\" +" subdir=("${prim_test_subdirs[$i]}") cd_vrfy "${test_configs_basedir}/$subdir" # @@ -931,16 +1024,121 @@ ${test_desc}${stripped_line} " # # First remove leading whitespace. # - test_desc="${test_desc#"${test_desc%%[![:space:]]*}"}" + test_desc="${test_desc#"${test_desc%%[![:space:]]*}"}" # # Now remove trailing whitespace. # - test_desc="${test_desc%"${test_desc##*[![:space:]]}"}" + test_desc="${test_desc%"${test_desc##*[![:space:]]}"}" # # Finally, save the description of the current test as the next element # of the array prim_test_descs. # prim_test_descs+=("${test_desc}") +# +# Get from the current test's configuration file the values of the +# variables specified in "vars_to_extract". Then save the value in the +# arrays specified by "prim_array_names_vars_to_extract". +# + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + + var_name="${vars_to_extract[$k]}" + cmd=$( grep "^[ ]*${var_name}=" "${config_fn}" ) + eval $cmd + + if [ -z "${!var_name+x}" ]; then + + msg=" + The variable \"${var_name}\" is not defined in the current test's + configuration file (config_fn): + config_fn = \"${config_fn}\" + Setting the element in the array \"${prim_array_names_vars_to_extract[$k]}\" + corresponding to this test to" + + case "${var_name}" in + + "NUM_ENS_MEMBERS") + default_val="1" + msg=$msg": + ${var_name} = \"${default_val}\"" + ;; + + "INCR_CYCL_FREQ") + default_val="24" + msg=$msg": + ${var_name} = \"${default_val}\"" + ;; + + *) + default_val="" + msg=$msg" an empty string." + ;; + + esac + cmd="${var_name}=\"${default_val}\"" + eval $cmd + + print_info_msg "$verbose" "$msg" + cmd="${prim_array_names_vars_to_extract[$k]}+=(\"'${default_val}\")" + + else +# +# The following are important notes regarding how the variable "cmd" +# containing the command that will append an element to the array +# specified by ${prim_array_names_vars_to_extract[$k]} is formulated: +# +# 1) If all the experiment variables were scalars, then the more complex +# command below could be replaced with the following: +# +# cmd="${prim_array_names_vars_to_extract[$k]}+=(\"${!var_name}\")" +# +# But some variables are arrays, so we need the more complex approach +# to cover those cases. +# +# 2) The double quotes (which need to be escaped here, i.e. \") are needed +# so that for any experiment variables that are arrays, all the elements +# of the array are combined together and treated as a single element. +# If the experiment variable is CYCL_HRS (cycle hours) and is set to +# the array ("00" "12"), we want the value saved in the local array +# here to be a single element consisting of "00 12". Otherwise, "00" +# and "12" will be treated as separate elements, and more than one +# element would be added to the array (which would be incorrect here). +# +# 3) The single quote (which needs to be escaped here, i.e. \') is needed +# so that any numbers (e.g. a set of cycle hours such as "00 12") are +# treated as strings when the CSV file is opened in Google Sheets. +# If this is not done, Google Sheets will remove leading zeros. +# + var_name_at="${var_name}[@]" + cmd="${prim_array_names_vars_to_extract[$k]}+=(\'\"${!var_name_at}\")" + fi + eval $cmd + + done +# +# Calculate the number of forecasts that will be launched by the current +# test. The "10#" forces bash to treat the following number as a decimal +# (not hexadecimal, etc). +# + num_cycles_per_day=${#CYCL_HRS[@]} + num_days=$(( (${DATE_LAST_CYCL} - ${DATE_FIRST_CYCL} + 1)*24/10#${INCR_CYCL_FREQ} )) + num_cdates=$(( ${num_cycles_per_day}*${num_days} )) + nf=$(( ${num_cdates}*10#${NUM_ENS_MEMBERS} )) +# +# In the following, the single quote at the beginning forces Google Sheets +# to interpret this quantity as a string. This prevents any automatic +# number fomatting from being applied when the CSV file is imported into +# Google Sheets. +# + prim_test_num_fcsts+=( "'$nf" ) +# +# Unset the experiment variables defined for the current test so that +# they are not accidentally used for the next one. +# + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + var_name="${vars_to_extract[$k]}" + cmd="unset ${var_name}" + eval $cmd + done done @@ -950,13 +1148,20 @@ ${test_desc}${stripped_line} " # # Create the arrays test_ids and test_descs that initially contain the # test IDs and descriptions corresponding to the primary test names -# (those of the alternate test names will be appended below). +# (those of the alternate test names will be appended below). Then, in +# the for-loop, do same for the arrays containing the experiment variable +# values for each test. # #----------------------------------------------------------------------- # test_ids=("${prim_test_ids[@]}") if [ "${get_test_descs}" = "TRUE" ]; then test_descs=("${prim_test_descs[@]}") + num_fcsts=("${prim_test_num_fcsts[@]}") + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + cmd="${array_names_vars_to_extract[$k]}=(\"\${${prim_array_names_vars_to_extract[$k]}[@]}\")" + eval $cmd + done fi # #----------------------------------------------------------------------- @@ -964,7 +1169,8 @@ ${test_desc}${stripped_line} " # Append to the arrays test_ids and test_descs the test IDs and descriptions # of the alternate test names. We set the test ID and description of # each alternate test name to those of the corresponding primary test -# name. +# name. Then, in the inner for-loop, do the same for the arrays containing +# the experiment variable values. # #----------------------------------------------------------------------- # @@ -980,6 +1186,11 @@ ${test_desc}${stripped_line} " test_ids+=("${prim_test_ids[$j]}") if [ "${get_test_descs}" = "TRUE" ]; then test_descs+=("${prim_test_descs[$j]}") + num_fcsts+=("${prim_test_num_fcsts[$j]}") + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + cmd="${array_names_vars_to_extract[$k]}+=(\"\${${prim_array_names_vars_to_extract[$k]}[$j]}\")" + eval $cmd + done fi num_occurrences=$((num_occurrences+1)) fi @@ -1003,7 +1214,9 @@ Please correct and rerun." #----------------------------------------------------------------------- # # Sort in order of increasing test ID the arrays containing the names, -# IDs, category subdirectories, and descriptions of the WE2E tests. +# IDs, category subdirectories, and descriptions of the WE2E tests as +# well as the arrays containing the experiment variable values for each +# test. # # For this purpose, we first create an array (test_ids_and_inds) each # of whose elements consist of the test ID, the test type, and the index @@ -1029,8 +1242,8 @@ Please correct and rerun." # and the test type, which we no longer need), which is the original # array index before sorting, and save the results in the array sort_inds. # This array will contain the original indices in sorted order that we -# then use to sort the arrays containing the names, IDs, subdirectories, -# and descriptions of the WE2E tests. +# then use to sort the arrays containing the WE2E test names, IDs, +# subdirectories, descriptions, and experiment variable values. # #----------------------------------------------------------------------- # @@ -1064,11 +1277,24 @@ Please correct and rerun." done if [ "${get_test_descs}" = "TRUE" ]; then + test_descs_orig=( "${test_descs[@]}" ) + num_fcsts_orig=( "${num_fcsts[@]}" ) + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + cmd="${array_names_vars_to_extract[$k]}_orig=(\"\${${array_names_vars_to_extract[$k]}[@]}\")" + eval $cmd + done + for (( i=0; i<=$((num_tests-1)); i++ )); do ii="${sort_inds[$i]}" test_descs[$i]="${test_descs_orig[$ii]}" + num_fcsts[$i]="${num_fcsts_orig[$ii]}" + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + cmd="${array_names_vars_to_extract[$k]}[$i]=\"\${${array_names_vars_to_extract[$k]}_orig[$ii]}\"" + eval $cmd + done done + fi # #----------------------------------------------------------------------- @@ -1094,18 +1320,27 @@ Please correct and rerun." # csv_delimiter="|" # -# Set the titles of the three columns that will be in the file. Then -# write them to the file. The contents of the columns are described in -# more detail further below. +# Set the titles of the columns that will be in the file. Then write +# them to the file. The contents of the columns are described in more +# detail further below. # column_titles="\ \"Test Name (Subdirectory)\" ${csv_delimiter} \ \"Alternate Test Names (Subdirectories)\" ${csv_delimiter} \ -\"Test Purpose/Description\"" +\"Test Purpose/Description\" ${csv_delimiter} \ +\"Number of Forecast Model Runs\"" + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + column_titles="\ +${column_titles} ${csv_delimiter} \ +\"${vars_to_extract[$k]}\"" + done printf "%s\n" "${column_titles}" >> "${csv_fp}" # # Loop through the arrays containing the WE2E test information. Extract # the necessary information and record it to the CSV file row-by-row. +# Note that each row corresponds to a primary test. When an alternate +# test is encountered, its information is stored in the row of the +# corresponding primary test (i.e. a new row is not created). # j=0 jp1=$((j+1)) @@ -1130,6 +1365,11 @@ Please correct and rerun." # test_desc=$( printf "%s" "${test_desc}" | sed -r -e "s/\"/\"\"/g" ) # +# Get the number of forecasts (number of times the forcast model is run, +# due to a unique starting date, an ensemble member, etc). +# + nf="${num_fcsts[$j]}" +# # In the following inner while-loop, we step through all alternate test # names (if any) that follow the current primary name and construct a # string (alt_test_names_subdirs) consisting of all the alternate test @@ -1167,11 +1407,30 @@ ${test_names[$jp1]} (${test_subdirs[$jp1]})" # # Column 3: # The test description. +# +# Column 4: +# The number of times the forecast model will be run by the test. This +# has been calculated above using the quantities that go in Columns 5, +# 6, .... +# +# Columns 5...: +# The values of the experiment variables specified in vars_to_extract. # row_content="\ \"${prim_test_name_subdir}\" ${csv_delimiter} \ \"${alt_test_names_subdirs}\" ${csv_delimiter} \ -\"${test_desc}\"" +\"${test_desc}\" ${csv_delimiter} \ +\"${nf}\"" + + for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + unset "val" + cmd="val=\"\${${array_names_vars_to_extract[$k]}[$j]}\"" + eval $cmd + row_content="\ +${row_content} ${csv_delimiter} \ +\"${val}\"" + done + printf "%s\n" "${row_content}" >> "${csv_fp}" # # Update loop indices. @@ -1181,6 +1440,11 @@ ${test_names[$jp1]} (${test_subdirs[$jp1]})" done + print_info_msg "\ +Successfully generated a CSV (Comma Separated Value) file (csv_fp) +containing information on all WE2E tests: + csv_fp = \"${csv_fp}\"" + fi # #----------------------------------------------------------------------- diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh index 53ce13a094..7476de7792 100644 --- a/ush/check_expt_config_vars.sh +++ b/ush/check_expt_config_vars.sh @@ -63,7 +63,7 @@ function check_expt_config_vars() { # Note that a variable name will be found only if the equal sign immediately # follows the variable name. # - var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") + var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p" ) if [ -z "${var_name}" ]; then diff --git a/ush/setup.sh b/ush/setup.sh index 26dd58b17f..77f674acd7 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -507,7 +507,7 @@ if [ -z "${NCORES_PER_NODE:-}" ]; then fi if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then - print_err_msg_exit "\ + print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} @@ -2163,7 +2163,6 @@ GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # variable definitions file. # #----------------------------------------------------------------------- - # print_info_msg " Creating list of default experiment variable definitions..." From c46b301713039ddf7a6912e79db0b8764c46c1cb Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 25 Apr 2022 09:23:18 -0400 Subject: [PATCH 155/203] Update directory structure of NCO mode (#743) * update vertical structure of NCO mode * update sample script for nco * Fix typo on write component of new RRFS CONUS --- tests/WE2E/run_WE2E_tests.sh | 32 ++++----- ...km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh | 1 - ...3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh | 2 - ush/config.nco.sh | 29 ++++---- ush/config_defaults.sh | 66 +++++++++---------- ush/machine/cheyenne.sh | 2 +- ush/machine/hera.sh | 2 +- ush/machine/jet.sh | 2 +- ush/machine/orion.sh | 2 +- ush/machine/wcoss_dell_p3.sh | 2 +- ush/set_extrn_mdl_params.sh | 6 +- ush/set_predef_grid_params.sh | 6 +- ush/setup.sh | 14 ++-- 13 files changed, 77 insertions(+), 89 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index f3040208ed..bcc3b0a6a6 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -854,7 +854,7 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # 2) The directory in which the output files from the post-processor (UPP) # for a given cycle are stored. The path to this directory is # -# \$PTMP/com/\$NET/\$envir/\$RUN.\$yyyymmdd/\$hh +# \$PTMP/com/\$NET/\$model_ver/\$RUN.\$yyyymmdd/\$hh # # Here, we make the first directory listed above unique to a WE2E test # by setting RUN to the name of the current test. This will also make @@ -872,40 +872,32 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # envir to the same value as RUN (which is just EXPT_SUBDIR). Then, for # this test, the UPP output will be located in the directory # -# \$PTMP/com/\$NET/\$RUN/\$RUN.\$yyyymmdd/\$hh +# \$PTMP/com/\$NET/\we2e/\$RUN.\$yyyymmdd/\$hh # RUN=\"\${EXPT_SUBDIR}\" -envir=\"\${EXPT_SUBDIR}\"" +model_ver="we2e"" # -# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model -# for ICs or LBCs. -# - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then +# Set COMIN. - COMINgfs=${TEST_COMINgfs:-} + COMIN=${TEST_COMIN:-} - if [ ! -d "${COMINgfs:-}" ] ; then - print_err_msg_exit "\ -The directory (COMINgfs) that needs to be specified when running the + if [ ! -d "${COMIN:-}" ] ; then + print_err_msg_exit "\ +The directory (COMIN) that needs to be specified when running the workflow in NCO mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as the external model for ICs and/or LBCs has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" - fi + fi - expt_config_str=${expt_config_str}" + expt_config_str=${expt_config_str}" # # Directory that needs to be specified when running the workflow in NCO -# mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or the GSMGFS as -# the external model for ICs and/or LBCs. +# mode (RUN_ENVIR set to \"nco\"). # -COMINgfs=\"${COMINgfs}\"" +COMIN=\"${COMIN}\"" - fi # # Set STMP and PTMP. # diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index f9eea07295..f6d6454bde 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -17,7 +17,6 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh index 6a02d4f119..19b36f87f5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.sh @@ -20,8 +20,6 @@ FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" - DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" CYCL_HRS=( "00" ) diff --git a/ush/config.nco.sh b/ush/config.nco.sh index 3b2c84be68..2e136c773c 100644 --- a/ush/config.nco.sh +++ b/ush/config.nco.sh @@ -8,27 +8,34 @@ VERBOSE="TRUE" RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" -PREDEF_GRID_NAME="CONUS_25km_GFDLgrid" +PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" +CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="06" -LBC_SPEC_INTVL_HRS="6" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" -DATE_FIRST_CYCL="20190901" -DATE_LAST_CYCL="20190901" -CYCL_HRS=( "18" ) +DATE_FIRST_CYCL="20220407" +DATE_LAST_CYCL="20220407" +CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" + +WRITE_DOPOST="TRUE" + # # The following must be modified for different platforms and users. # -RUN="an_experiment" -COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" # Path to directory containing files from the external model (FV3GFS). +NET="rrfs" +model_ver="v1.0" +RUN="rrfs_test" +COMIN="/scratch1/NCEPDEV/rstprod/com/gfs/prod" # Path to directory containing files from the external model. FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" # Path to directory containing the pregenerated grid, orography, and surface climatology "fixed" files to use for the experiment. -STMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/stmp" # Path to directory STMP that mostly contains input files. -PTMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/ptmp" # Path to directory PTMP in which the experiment's output files will be placed. +STMP="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/01_OUT_DATA/stmp" # Path to directory STMP that mostly contains input files. +PTMP="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/01_OUT_DATA/ptmp" # Path to directory PTMP in which the experiment's output files will be placed. diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 9345704876..0649fcbb3e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -17,8 +17,8 @@ # # NCEP Central Operations # WCOSS Implementation Standards -# April 17, 2019 -# Version 10.2.0 +# January 19, 2022 +# Version 11.0.0 # # RUN_ENVIR is described in this document as follows: # @@ -232,18 +232,11 @@ EXEC_SUBDIR="bin" # Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is # set to "nco"). Definitions: # -# COMINgfs: -# The beginning portion of the directory containing files generated by -# the external model (FV3GFS) that the initial and lateral boundary -# condition generation tasks need in order to create initial and boundary -# condition files for a given cycle on the native FV3-LAM grid. For a -# cycle that starts on the date specified by the variable yyyymmdd -# (consisting of the 4-digit year followed by the 2-digit month followed -# by the 2-digit day of the month) and hour specified by the variable hh -# (consisting of the 2-digit hour-of-day), the directory in which the -# workflow will look for the external model files is: -# -# $COMINgfs/gfs.$yyyymmdd/$hh/atmos +# COMIN: +# Directory containing files generated by the external model (FV3GFS, NAM, +# HRRR, etc) that the initial and lateral boundary condition generation tasks +# need in order to create initial and boundary condition files for a given +# cycle on the native FV3-LAM grid. # # FIXLAM_NCO_BASEDIR: # The base directory containing pregenerated grid, orography, and surface @@ -258,6 +251,24 @@ EXEC_SUBDIR="bin" # string in this file, but it can be specified in the user-specified # workflow configuration file (EXPT_CONFIG_FN). # +# envir, NET, model_ver, RUN: +# Standard environment variables defined in the NCEP Central Operations WCOSS +# Implementation Standards document as follows: +# +# envir: +# Set to "test" during the initial testing phase, "para" when running +# in parallel (on a schedule), and "prod" in production. +# +# NET: +# Model name (first level of com directory structure) +# +# model_ver: +# Version number of package in three digits (second level of com directory) +# +# RUN: +# Name of model run (third level of com directory structure). +# In general, same as $NET +# # STMP: # The beginning portion of the directory that will contain cycle-dependent # model input files, symlinks to cycle-independent input files, and raw @@ -269,22 +280,6 @@ EXEC_SUBDIR="bin" # # $STMP/tmpnwprd/$RUN/$cdate # -# NET, envir, RUN: -# Variables used in forming the path to the directory that will contain -# the output files from the post-processor (UPP) for a given cycle (see -# definition of PTMP below). These are defined in the WCOSS Implementation -# Standards document as follows: -# -# NET: -# Model name (first level of com directory structure) -# -# envir: -# Set to "test" during the initial testing phase, "para" when running -# in parallel (on a schedule), and "prod" in production. -# -# RUN: -# Name of model run (third level of com directory structure). -# # PTMP: # The beginning portion of the directory that will contain the output # files from the post-processor (UPP) for a given cycle. For a cycle @@ -292,16 +287,17 @@ EXEC_SUBDIR="bin" # (where yyyymmdd and hh are as described above), the directory in which # the UPP output files will be placed will be: # -# $PTMP/com/$NET/$envir/$RUN.$yyyymmdd/$hh +# $PTMP/com/$NET/$model_ver/$RUN.$yyyymmdd/$hh # #----------------------------------------------------------------------- # -COMINgfs="/base/path/of/directory/containing/gfs/input/files" +COMIN="/path/of/directory/containing/data/files/for/IC/LBCS" FIXLAM_NCO_BASEDIR="" -STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" -NET="rrfs" envir="para" -RUN="experiment_name" +NET="rrfs" +model_ver="v1.0.0" +RUN="rrfs" +STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" PTMP="/base/path/of/directory/containing/postprocessed/output/files" # #----------------------------------------------------------------------- diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index 6cf61061a0..f92c6fa844 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -69,5 +69,5 @@ MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/FV3LAM_pregen" -TEST_COMINgfs="/glade/p/ral/jntp/UFS_SRW_app/COMGFS" +TEST_COMIN="/glade/p/ral/jntp/UFS_SRW_app/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index bb0f773134..39c28f8ee4 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -72,7 +72,7 @@ MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" -TEST_COMINgfs="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" +TEST_COMIN="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/develop/model_data" TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index dac90d5490..9a12c0c142 100644 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -80,5 +80,5 @@ RUN_CMD_POST="srun" # Test Data Locations TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/FV3LAM_pregen" -TEST_COMINgfs="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/COMGFS" +TEST_COMIN="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/staged_extrn_mdl_files" diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index 5c872410d4..ce39308d4e 100644 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -66,6 +66,6 @@ MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations TEST_PREGEN_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/FV3LAM_pregen" -TEST_COMINgfs="/work/noaa/fv3-cam/UFS_SRW_App/COMGFS" +TEST_COMIN="/work/noaa/fv3-cam/UFS_SRW_App/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/develop/model_data" diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 811c3ae8b2..0c8b7cd248 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -78,5 +78,5 @@ MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} # Test Data Locations TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" -TEST_COMINgfs="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" +TEST_COMIN="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/model_data" diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index f110a4c26e..9664d6a471 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -9,13 +9,13 @@ function set_extrn_mdl_params() { # #----------------------------------------------------------------------- # - # Use known locations or COMINgfs as default, depending on RUN_ENVIR + # Use known locations or COMIN as default, depending on RUN_ENVIR # #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMINgfs}" - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMINgfs}" + EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMIN}" + EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMIN}" else EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$(set_known_sys_dir \ ${EXTRN_MDL_NAME_ICS})}" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 88b2c0f633..d14e9381f7 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -132,7 +132,7 @@ case ${PREDEF_GRID_NAME} in WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" WRTCMP_nx="217" WRTCMP_ny="128" - WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lon_lwr_left="-122.719528" WRTCMP_lat_lwr_left="21.138123" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" @@ -224,7 +224,7 @@ case ${PREDEF_GRID_NAME} in WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" WRTCMP_nx="416" WRTCMP_ny="245" - WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lon_lwr_left="-122.719528" WRTCMP_lat_lwr_left="21.138123" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" @@ -316,7 +316,7 @@ case ${PREDEF_GRID_NAME} in WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" WRTCMP_nx="1799" WRTCMP_ny="1059" - WRTCMP_lon_lwr_left="-122.719258" + WRTCMP_lon_lwr_left="-122.719528" WRTCMP_lat_lwr_left="21.138123" WRTCMP_dx="${ESGgrid_DELX}" WRTCMP_dy="${ESGgrid_DELY}" diff --git a/ush/setup.sh b/ush/setup.sh index 77f674acd7..835ccfea48 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1141,7 +1141,7 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # is not placed directly under COMROOT but several directories further # down. More specifically, for a cycle starting at yyyymmddhh, it is at # -# $COMROOT/$NET/$envir/$RUN.$yyyymmdd/$hh +# $COMROOT/$NET/$model_ver/$RUN.$yyyymmdd/$hh # # Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMOROOT # is not used by the workflow in community mode. @@ -1151,7 +1151,7 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # from the RUN_POST_TN task will be placed, i.e. it is the cycle-independent # portion of the RUN_POST_TN task's output directory. It is given by # -# $COMROOT/$NET/$envir +# $COMROOT/$NET/$model_ver # # COMOUT_BASEDIR is not used by the workflow in community mode. # @@ -1164,19 +1164,15 @@ FIXclim="${EXPTDIR}/fix_clim" FIXLAM="${EXPTDIR}/fix_lam" if [ "${RUN_ENVIR}" = "nco" ]; then - - CYCLE_BASEDIR="$STMP/tmpnwprd/$RUN" + CYCLE_BASEDIR="${STMP}/tmpnwprd/${RUN}" check_for_preexist_dir_file "${CYCLE_BASEDIR}" "${PREEXISTING_DIR_METHOD}" - COMROOT="$PTMP/com" - COMOUT_BASEDIR="$COMROOT/$NET/$envir" + COMROOT="${PTMP}/com" + COMOUT_BASEDIR="${COMROOT}/${NET}/${model_ver}" check_for_preexist_dir_file "${COMOUT_BASEDIR}" "${PREEXISTING_DIR_METHOD}" - else - CYCLE_BASEDIR="$EXPTDIR" COMROOT="" COMOUT_BASEDIR="" - fi # #----------------------------------------------------------------------- From 5fffa023d66da9baa607a3f18c01ec7877823fd1 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:58:09 -0400 Subject: [PATCH 156/203] Default CCPP physics option is FV3_GFS_v16 (#746) * Updated the default CCPP physics option to FV3_GFS_v16 * Updated the default CCPP physics option to FV3_GFS_v16 in config_defaults.sh Co-authored-by: Natalie Perlin --- ush/config.community.sh | 4 ++-- ush/config_defaults.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/config.community.sh b/ush/config.community.sh index 70433d5973..ba8aa44da2 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -14,7 +14,7 @@ QUILTING="TRUE" DO_ENSEMBLE="FALSE" NUM_ENS_MEMBERS="2" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" +CCPP_PHYS_SUITE="FV3_GFS_v16" FCST_LEN_HRS="48" LBC_SPEC_INTVL_HRS="6" @@ -30,7 +30,7 @@ FV3GFS_FILE_FMT_LBCS="grib2" WTIME_RUN_FCST="01:00:00" -MODEL="FV3_GFS_v15p2_CONUS_25km" +MODEL="FV3_GFS_v16_CONUS_25km" METPLUS_PATH="path/to/METPlus" MET_INSTALL_DIR="path/to/MET" CCPA_OBS_DIR="/path/to/processed/CCPA/data" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 0649fcbb3e..ba57dca9d9 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -747,7 +747,7 @@ NOMADS_file_type="nemsio" # #----------------------------------------------------------------------- # -CCPP_PHYS_SUITE="FV3_GFS_v15p2" +CCPP_PHYS_SUITE="FV3_GFS_v16" # #----------------------------------------------------------------------- # From 0ea64f637b241eb174d5846333a770a75e23cb06 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Mon, 25 Apr 2022 20:43:29 -0600 Subject: [PATCH 157/203] Adds an alternative python workflow generation path (#698) * Workflow in python starting to work. * Use new python_utils package structure. * Some bug fixes. * Use uppercase TRUE/FALSE in var_dfns * Use config.sh by default. * Minor bug fixes. * Remove config.yaml * Update to the latest develop * Remove quotes from numbers in predef grid. * Minor bug fix. * Move validity checker to the bottom of setup * Add more unit tests. * Update with python_utils changes. * Update to latest develop additions (Need to re-run regression test) * Use set_namelist and fill_jinja_template as python functions. * Replace sed regex searches with python re. * Use python realpath. * Construct settings as dictionary before passing to fill_jinja and set_namelist * Use yaml for setting predefined grid parameters. * Use xml parser for ccpp phys suite definition file. * Remove more run_command calls. * Simplify some func argument processing. * Move different config format parsers to same file. * Use os.path.join for the sake of macosx * Remove remaining func argument processing via os.environ. * Minor bug fix in set_extrn_mdl_params.sh * Add suite defn in test_data. * Minor fixes on unittest on jet. * Simplify boolean condition checks. * Include old in renaming of old directories * Fix conflicting yaml !join tag for paths and strings. * Bug fix with setting sfcperst dict. * Imitate "readlink -m" with os.path.realpath instead of os.readlink * Don't use /tmp as that is shared by multiple users. * Bug fix with cron line, maintain quotes around TRUE/FALSE. * Update to latest develop (untested) * Bug fix with existing cron line and quotes. * Bug fix with case-sensitive MACHINE name, and empty EXPT_DIR. * Update to latest develop * More updates. * Bug fix thanks to @willmayfield! Check both starting/ending characters are brackets for shell variable to be considered an array. * Make empty EXPT_BASEDIR workable. * Update to latest develop * Update in predef grid. * Check f90nml as well. Co-authored-by: Daniel Abdi --- ush/check_ruc_lsm.py | 30 + ush/config_defaults.yaml | 1998 +++++++++++++++ ush/constants.py | 27 + ush/create_diag_table_file.py | 79 + ush/create_model_configure_file.py | 243 ++ ush/fill_jinja_template.py | 18 +- ush/generate_FV3LAM_wflow.py | 1126 +++++++++ ush/get_crontab_contents.py | 83 + ush/link_fix.py | 391 +++ ush/predef_grid_params.yaml | 866 +++++++ ush/python_utils/__init__.py | 12 +- ush/python_utils/change_case.py | 25 - .../check_for_preexist_dir_file.py | 2 +- ush/python_utils/config_parser.py | 159 +- ush/python_utils/environment.py | 2 +- ush/python_utils/get_elem_inds.py | 2 +- .../get_manage_externals_config_property.py | 54 - ush/python_utils/misc.py | 57 + ush/python_utils/print_input_args.py | 2 +- ush/python_utils/test_python_utils.py | 45 +- ush/python_utils/xml_parser.py | 30 + ush/set_FV3nml_ens_stoch_seeds.py | 137 + ush/set_FV3nml_sfc_climo_filenames.py | 141 ++ ush/set_cycle_dates.py | 54 + ush/set_extrn_mdl_params.py | 79 + ush/set_extrn_mdl_params.sh | 2 +- ush/set_gridparams_ESGgrid.py | 110 + ush/set_gridparams_GFDLgrid.py | 467 ++++ ush/set_namelist.py | 17 +- ush/set_ozone_param.py | 231 ++ ush/set_predef_grid_params.py | 59 + ush/set_thompson_mp_fix_files.py | 176 ++ ush/setup.py | 2195 +++++++++++++++++ .../RRFS_CONUS_3km/C3357.facsf.tile7.halo0.nc | 0 .../RRFS_CONUS_3km/C3357.facsf.tile7.halo4.nc | 0 .../C3357.maximum_snow_albedo.tile7.halo0.nc | 0 .../C3357.maximum_snow_albedo.tile7.halo4.nc | 0 .../C3357.slope_type.tile7.halo0.nc | 0 .../C3357.slope_type.tile7.halo4.nc | 0 .../C3357.snowfree_albedo.tile7.halo0.nc | 0 .../C3357.snowfree_albedo.tile7.halo4.nc | 0 .../C3357.soil_type.tile7.halo0.nc | 0 .../C3357.soil_type.tile7.halo4.nc | 0 ...C3357.substrate_temperature.tile7.halo0.nc | 0 ...C3357.substrate_temperature.tile7.halo4.nc | 0 .../C3357.vegetation_greenness.tile7.halo0.nc | 0 .../C3357.vegetation_greenness.tile7.halo4.nc | 0 .../C3357.vegetation_type.tile7.halo0.nc | 0 .../C3357.vegetation_type.tile7.halo4.nc | 0 .../RRFS_CONUS_3km/C3357_grid.tile7.halo3.nc | 0 .../RRFS_CONUS_3km/C3357_grid.tile7.halo4.nc | 0 .../RRFS_CONUS_3km/C3357_grid.tile7.halo6.nc | 0 .../RRFS_CONUS_3km/C3357_mosaic.halo3.nc | 0 .../RRFS_CONUS_3km/C3357_mosaic.halo4.nc | 0 .../RRFS_CONUS_3km/C3357_mosaic.halo6.nc | 0 .../C3357_oro_data.tile7.halo0.nc | 0 .../C3357_oro_data.tile7.halo4.nc | 0 .../C3357_oro_data_ls.tile7.halo0.nc | 0 .../C3357_oro_data_ss.tile7.halo0.nc | 0 ush/test_data/suite_FV3_GSD_SAR.xml | 85 + ush/valid_param_vals.yaml | 84 + 61 files changed, 8946 insertions(+), 142 deletions(-) create mode 100644 ush/check_ruc_lsm.py create mode 100644 ush/config_defaults.yaml create mode 100644 ush/constants.py create mode 100644 ush/create_diag_table_file.py create mode 100644 ush/create_model_configure_file.py create mode 100755 ush/generate_FV3LAM_wflow.py create mode 100644 ush/get_crontab_contents.py create mode 100644 ush/link_fix.py create mode 100644 ush/predef_grid_params.yaml delete mode 100644 ush/python_utils/change_case.py delete mode 100644 ush/python_utils/get_manage_externals_config_property.py create mode 100644 ush/python_utils/misc.py create mode 100644 ush/python_utils/xml_parser.py create mode 100644 ush/set_FV3nml_ens_stoch_seeds.py create mode 100644 ush/set_FV3nml_sfc_climo_filenames.py create mode 100644 ush/set_cycle_dates.py create mode 100644 ush/set_extrn_mdl_params.py create mode 100644 ush/set_gridparams_ESGgrid.py create mode 100644 ush/set_gridparams_GFDLgrid.py create mode 100644 ush/set_ozone_param.py create mode 100644 ush/set_predef_grid_params.py create mode 100644 ush/set_thompson_mp_fix_files.py create mode 100644 ush/setup.py create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo3.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo6.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo3.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo6.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo4.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ls.tile7.halo0.nc create mode 100644 ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ss.tile7.halo0.nc create mode 100644 ush/test_data/suite_FV3_GSD_SAR.xml create mode 100644 ush/valid_param_vals.yaml diff --git a/ush/check_ruc_lsm.py b/ush/check_ruc_lsm.py new file mode 100644 index 0000000000..fdf288f30a --- /dev/null +++ b/ush/check_ruc_lsm.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import os +import unittest + +from python_utils import set_env_var, print_input_args, \ + load_xml_file, has_tag_with_value + +def check_ruc_lsm(ccpp_phys_suite_fp): + """ This file defines a function that checks whether the RUC land surface + model (LSM) parameterization is being called by the selected physics suite. + + Args: + ccpp_phys_suite_fp: full path to CCPP physics suite xml file + Returns: + Boolean + """ + + print_input_args(locals()) + + tree = load_xml_file(ccpp_phys_suite_fp) + has_ruc = has_tag_with_value(tree, "scheme", "lsm_ruc") + return has_ruc + +class Testing(unittest.TestCase): + def test_check_ruc_lsm(self): + self.assertTrue( check_ruc_lsm(ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml") ) + def setUp(self): + set_env_var('DEBUG',True) + diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml new file mode 100644 index 0000000000..ac35529530 --- /dev/null +++ b/ush/config_defaults.yaml @@ -0,0 +1,1998 @@ +# +#----------------------------------------------------------------------- +# +# This file sets the experiment's configuration variables (which are +# global shell variables) to their default values. For many of these +# variables, the valid values that they may take on are defined in the +# file $USHDIR/valid_param_vals.sh. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Set the RUN_ENVIR variable that is listed and described in the WCOSS +# Implementation Standards document: +# +# NCEP Central Operations +# WCOSS Implementation Standards +# April 19, 2022 +# Version 11.0.0 +# +# RUN_ENVIR is described in this document as follows: +# +# Set to "nco" if running in NCO's production environment. Used to +# distinguish between organizations. +# +# Valid values are "nco" and "community". Here, we use it to generate +# and run the experiment either in NCO mode (if RUN_ENVIR is set to "nco") +# or in community mode (if RUN_ENVIR is set to "community"). This has +# implications on the experiment variables that need to be set and the +# the directory structure used. +# +#----------------------------------------------------------------------- +# +RUN_ENVIR: "nco" +# +#----------------------------------------------------------------------- +# +# mach_doc_start +# Set machine and queue parameters. Definitions: +# +# MACHINE: +# Machine on which the workflow will run. If you are NOT on a named, +# supported platform, and you want to use the Rocoto workflow manager, +# you will need set MACHINE: "linux" and WORKFLOW_MANAGER: "rocoto". This +# combination will assume a Slurm batch manager when generating the XML. +# Please see ush/valid_param_vals.sh for a full list of supported +# platforms. +# +# MACHINE_FILE: +# Path to a configuration file with machine-specific settings. If none +# is provided, setup.sh will attempt to set the path to for a supported +# platform. +# +# ACCOUNT: +# The account under which to submit jobs to the queue. +# +# WORKFLOW_MANAGER: +# The workflow manager to use (e.g. rocoto). This is set to "none" by +# default, but if the machine name is set to a platform that supports +# rocoto, this will be overwritten and set to "rocoto". If set +# explicitly to rocoto along with the use of the MACHINE=linux target, +# the configuration layer assumes a Slurm batch manager when generating +# the XML. Valid options: "rocoto" or "none" +# +# NCORES_PER_NODE: +# The number of cores available per node on the compute platform. Set +# for supported platforms in setup.sh, but is now also configurable for +# all platforms. +# +# LMOD_PATH: +# Path to the LMOD sh file on your Linux system. Is set automatically +# for supported machines. +# +# BUILD_ENV_FN: +# Name of alternative build environment file to use if using an +# unsupported platform. Is set automatically for supported machines. +# +# WFLOW_ENV_FN: +# Name of alternative workflow environment file to use if using an +# unsupported platform. Is set automatically for supported machines. +# +# SCHED: +# The job scheduler to use (e.g. slurm). Set this to an empty string in +# order for the experiment generation script to set it depending on the +# machine. +# +# PARTITION_DEFAULT: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the default partition to which to submit workflow tasks. If a task +# does not have a specific variable that specifies the partition to which +# it will be submitted (e.g. PARTITION_HPSS, PARTITION_FCST; see below), +# it will be submitted to the partition specified by this variable. If +# this is not set or is set to an empty string, it will be (re)set to a +# machine-dependent value. This is not used if SCHED is not set to +# "slurm". +# +# QUEUE_DEFAULT: +# The default queue or QOS (if using the slurm job scheduler, where QOS +# is Quality of Service) to which workflow tasks are submitted. If a +# task does not have a specific variable that specifies the queue to which +# it will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will +# be submitted to the queue specified by this variable. If this is not +# set or is set to an empty string, it will be (re)set to a machine- +# dependent value. +# +# PARTITION_HPSS: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the partition to which the tasks that get or create links to external +# model files [which are needed to generate initial conditions (ICs) and +# lateral boundary conditions (LBCs)] are submitted. If this is not set +# or is set to an empty string, it will be (re)set to a machine-dependent +# value. This is not used if SCHED is not set to "slurm". +# +# QUEUE_HPSS: +# The queue or QOS to which the tasks that get or create links to external +# model files [which are needed to generate initial conditions (ICs) and +# lateral boundary conditions (LBCs)] are submitted. If this is not set +# or is set to an empty string, it will be (re)set to a machine-dependent +# value. +# +# PARTITION_FCST: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the partition to which the task that runs forecasts is submitted. If +# this is not set or set to an empty string, it will be (re)set to a +# machine-dependent value. This is not used if SCHED is not set to +# "slurm". +# +# QUEUE_FCST: +# The queue or QOS to which the task that runs a forecast is submitted. +# If this is not set or set to an empty string, it will be (re)set to a +# machine-dependent value. +# +# mach_doc_end +# +#----------------------------------------------------------------------- +# +MACHINE: "BIG_COMPUTER" +MACHINE_FILE: "" +ACCOUNT: "project_name" +WORKFLOW_MANAGER: "none" +NCORES_PER_NODE: "" +LMOD_PATH: "" +BUILD_ENV_FN: "" +WFLOW_ENV_FN: "" +SCHED: "" +PARTITION_DEFAULT: "" +QUEUE_DEFAULT: "" +PARTITION_HPSS: "" +QUEUE_HPSS: "" +PARTITION_FCST: "" +QUEUE_FCST: "" +# +#----------------------------------------------------------------------- +# +# Set run commands for platforms without a workflow manager. These values +# will be ignored unless WORKFLOW_MANAGER: "none". Definitions: +# +# RUN_CMD_UTILS: +# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, +# etc.) Can be left blank for smaller domains, in which case the executables +# will run without MPI. +# +# RUN_CMD_FCST: +# The run command for the model forecast step. This will be appended to +# the end of the variable definitions file, so it can reference other +# variables. +# +# RUN_CMD_POST: +# The run command for post-processing (UPP). Can be left blank for smaller +# domains, in which case UPP will run without MPI. +# +#----------------------------------------------------------------------- +# +RUN_CMD_UTILS: "mpirun -np 1" +RUN_CMD_FCST: "mpirun -np ${PE_MEMBER01}" +RUN_CMD_POST: "mpirun -np 1" +# +#----------------------------------------------------------------------- +# +# Set cron-associated parameters. Definitions: +# +# USE_CRON_TO_RELAUNCH: +# Flag that determines whether or not to add a line to the user's cron +# table to call the experiment launch script every CRON_RELAUNCH_INTVL_MNTS +# minutes. +# +# CRON_RELAUNCH_INTVL_MNTS: +# The interval (in minutes) between successive calls of the experiment +# launch script by a cron job to (re)launch the experiment (so that the +# workflow for the experiment kicks off where it left off). +# +#----------------------------------------------------------------------- +# +USE_CRON_TO_RELAUNCH: "FALSE" +CRON_RELAUNCH_INTVL_MNTS: "03" +# +#----------------------------------------------------------------------- +# +# dir_doc_start +# Set directories. Definitions: +# +# EXPT_BASEDIR: +# The base directory in which the experiment directory will be created. +# If this is not specified or if it is set to an empty string, it will +# default to ${HOMErrfs}/../expt_dirs. +# +# EXPT_SUBDIR: +# The name that the experiment directory (without the full path) will +# have. The full path to the experiment directory, which will be contained +# in the variable EXPTDIR, will be: +# +# EXPTDIR: "${EXPT_BASEDIR}/${EXPT_SUBDIR}" +# +# This cannot be empty. If set to a null string here, it must be set to +# a (non-empty) value in the user-defined experiment configuration file. +# +# dir_doc_end +# +# EXEC_SUBDIR: +# The name of the subdirectory of ufs-srweather-app where executables are +# installed. +#----------------------------------------------------------------------- +# +EXPT_BASEDIR: "" +EXPT_SUBDIR: "" +EXEC_SUBDIR: "bin" +# +#----------------------------------------------------------------------- +# +# Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is +# set to "nco"). Definitions: +# +# COMIN: +# Directory containing files generated by the external model (FV3GFS, NAM, +# HRRR, etc) that the initial and lateral boundary condition generation tasks +# need in order to create initial and boundary condition files for a given +# cycle on the native FV3-LAM grid. +# +# FIXLAM_NCO_BASEDIR: +# The base directory containing pregenerated grid, orography, and surface +# climatology files. For the pregenerated grid specified by PREDEF_GRID_NAME, +# these "fixed" files are located in: +# +# ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME} +# +# The workflow scripts will create a symlink in the experiment directory +# that will point to a subdirectory (having the name of the grid being +# used) under this directory. This variable should be set to a null +# string in this file, but it can be specified in the user-specified +# workflow configuration file (EXPT_CONFIG_FN). +# +# envir, NET, model_ver, RUN: +# Standard environment variables defined in the NCEP Central Operations WCOSS +# Implementation Standards document as follows: +# +# envir: +# Set to "test" during the initial testing phase, "para" when running +# in parallel (on a schedule), and "prod" in production. +# +# NET: +# Model name (first level of com directory structure) +# +# model_ver: +# Version number of package in three digits (second level of com directory) +# +# RUN: +# Name of model run (third level of com directory structure). +# In general, same as $NET +# +# STMP: +# The beginning portion of the directory that will contain cycle-dependent +# model input files, symlinks to cycle-independent input files, and raw +# (i.e. before post-processing) forecast output files for a given cycle. +# For a cycle that starts on the date specified by yyyymmdd and hour +# specified by hh (where yyyymmdd and hh are as described above) [so that +# the cycle date (cdate) is given by cdate: "${yyyymmdd}${hh}"], the +# directory in which the aforementioned files will be located is: +# +# $STMP/tmpnwprd/$RUN/$cdate +# +# PTMP: +# The beginning portion of the directory that will contain the output +# files from the post-processor (UPP) for a given cycle. For a cycle +# that starts on the date specified by yyyymmdd and hour specified by hh +# (where yyyymmdd and hh are as described above), the directory in which +# the UPP output files will be placed will be: +# +# $PTMP/com/$NET/$model_ver/$RUN.$yyyymmdd/$hh +# +#----------------------------------------------------------------------- +# +COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS" +FIXLAM_NCO_BASEDIR: "" +envir: "para" +NET: "rrfs" +model_ver: "v1.0.0" +RUN: "rrfs" +STMP: "/base/path/of/directory/containing/model/input/and/raw/output/files" +PTMP: "/base/path/of/directory/containing/postprocessed/output/files" +# +#----------------------------------------------------------------------- +# +# Set the separator character(s) to use in the names of the grid, mosaic, +# and orography fixed files. +# +# Ideally, the same separator should be used in the names of these fixed +# files as the surface climatology fixed files (which always use a "." +# as the separator), i.e. ideally, DOT_OR_USCORE should be set to "." +# +#----------------------------------------------------------------------- +# +DOT_OR_USCORE: "_" +# +#----------------------------------------------------------------------- +# +# Set file names. Definitions: +# +# EXPT_CONFIG_FN: +# Name of the user-specified configuration file for the forecast experiment. +# +# RGNL_GRID_NML_FN: +# Name of file containing the namelist settings for the code that generates +# a "ESGgrid" type of regional grid. +# +# FV3_NML_BASE_SUITE_FN: +# Name of Fortran namelist file containing the forecast model's base suite +# namelist, i.e. the portion of the namelist that is common to all physics +# suites. +# +# FV3_NML_YAML_CONFIG_FN: +# Name of YAML configuration file containing the forecast model's namelist +# settings for various physics suites. +# +# FV3_NML_BASE_ENS_FN: +# Name of Fortran namelist file containing the forecast model's base +# ensemble namelist, i.e. the the namelist file that is the starting point +# from which the namelist files for each of the enesemble members are +# generated. +# +# FV3_EXEC_FN: +# Name to use for the forecast model executable when it is copied from +# the directory in which it is created in the build step to the executables +# directory (EXECDIR; this is set during experiment generation). +# +# DIAG_TABLE_TMPL_FN: +# Name of a template file that specifies the output fields of the forecast +# model (ufs-weather-model: diag_table) followed by [dot_ccpp_phys_suite]. +# Its default value is the name of the file that the ufs weather model +# expects to read in. +# +# FIELD_TABLE_TMPL_FN: +# Name of a template file that specifies the tracers in IC/LBC files of the +# forecast model (ufs-weather-mode: field_table) followed by [dot_ccpp_phys_suite]. +# Its default value is the name of the file that the ufs weather model expects +# to read in. +# +# MODEL_CONFIG_TMPL_FN: +# Name of a template file that contains settings and configurations for the +# NUOPC/ESMF main component (ufs-weather-model: model_config). Its default +# value is the name of the file that the ufs weather model expects to read in. +# +# NEMS_CONFIG_TMPL_FN: +# Name of a template file that contains information about the various NEMS +# components and their run sequence (ufs-weather-model: nems.configure). +# Its default value is the name of the file that the ufs weather model expects +# to read in. +# +# FCST_MODEL: +# Name of forecast model (default=ufs-weather-model) +# +# WFLOW_XML_FN: +# Name of the rocoto workflow XML file that the experiment generation +# script creates and that defines the workflow for the experiment. +# +# GLOBAL_VAR_DEFNS_FN: +# Name of file (a shell script) containing the defintions of the primary +# experiment variables (parameters) defined in this default configuration +# script and in the user-specified configuration as well as secondary +# experiment variables generated by the experiment generation script. +# This file is sourced by many scripts (e.g. the J-job scripts corresponding +# to each workflow task) in order to make all the experiment variables +# available in those scripts. +# +# EXTRN_MDL_VAR_DEFNS_FN: +# Name of file (a shell script) containing the defintions of variables +# associated with the external model from which ICs or LBCs are generated. This +# file is created by the GET_EXTRN_*_TN task because the values of the variables +# it contains are not known before this task runs. The file is then sourced by +# the MAKE_ICS_TN and MAKE_LBCS_TN tasks. +# +# WFLOW_LAUNCH_SCRIPT_FN: +# Name of the script that can be used to (re)launch the experiment's rocoto +# workflow. +# +# WFLOW_LAUNCH_LOG_FN: +# Name of the log file that contains the output from successive calls to +# the workflow launch script (WFLOW_LAUNCH_SCRIPT_FN). +# +#----------------------------------------------------------------------- +# +EXPT_CONFIG_FN: "config.sh" + +RGNL_GRID_NML_FN: "regional_grid.nml" + +FV3_NML_BASE_SUITE_FN: "input.nml.FV3" +FV3_NML_YAML_CONFIG_FN: "FV3.input.yml" +FV3_NML_BASE_ENS_FN: "input.nml.base_ens" +FV3_EXEC_FN: "ufs_model" + +DATA_TABLE_TMPL_FN: "" +DIAG_TABLE_TMPL_FN: "" +FIELD_TABLE_TMPL_FN: "" +MODEL_CONFIG_TMPL_FN: "" +NEMS_CONFIG_TMPL_FN: "" + +FCST_MODEL: "ufs-weather-model" +WFLOW_XML_FN: "FV3LAM_wflow.xml" +GLOBAL_VAR_DEFNS_FN: "var_defns.sh" +EXTRN_MDL_VAR_DEFNS_FN: "extrn_mdl_var_defns.sh" +WFLOW_LAUNCH_SCRIPT_FN: "launch_FV3LAM_wflow.sh" +WFLOW_LAUNCH_LOG_FN: "log.launch_FV3LAM_wflow" +# +#----------------------------------------------------------------------- +# +# Set forecast parameters. Definitions: +# +# DATE_FIRST_CYCL: +# Starting date of the first forecast in the set of forecasts to run. +# Format is "YYYYMMDD". Note that this does not include the hour-of-day. +# +# DATE_LAST_CYCL: +# Starting date of the last forecast in the set of forecasts to run. +# Format is "YYYYMMDD". Note that this does not include the hour-of-day. +# +# CYCL_HRS: +# An array containing the hours of the day at which to launch forecasts. +# Forecasts are launched at these hours on each day from DATE_FIRST_CYCL +# to DATE_LAST_CYCL, inclusive. Each element of this array must be a +# two-digit string representing an integer that is less than or equal to +# 23, e.g. "00", "03", "12", "23". +# +# INCR_CYCL_FREQ: +# Increment in hours for Cycle Frequency (cycl_freq). +# Default is 24, which means cycle_freq=24:00:00 +# +# FCST_LEN_HRS: +# The length of each forecast, in integer hours. +# +#----------------------------------------------------------------------- +# +DATE_FIRST_CYCL: "YYYYMMDD" +DATE_LAST_CYCL: "YYYYMMDD" +CYCL_HRS: [ "HH1", "HH2" ] +INCR_CYCL_FREQ: "24" +FCST_LEN_HRS: "24" +# +#----------------------------------------------------------------------- +# +# Set model_configure parameters. Definitions: +# +# DT_ATMOS: +# The main forecast model integraton time step. As described in the +# forecast model documentation, "It corresponds to the frequency with +# which the top level routine in the dynamics is called as well as the +# frequency with which the physics is called." +# +# CPL: parameter for coupling +# (set automatically based on FCST_MODEL in ush/setup.sh) +# (ufs-weather-model:FALSE, fv3gfs_aqm:TRUE) +# +# RESTART_INTERVAL: +# frequency of the output restart files (unit:hour). +# Default=0: restart files are produced at the end of a forecast run +# For example, RESTART_INTERVAL: "1": restart files are produced every hour +# with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory +# +# WRITE_DOPOST: +# Flag that determines whether or not to use the INLINE POST option +# When TRUE, force to turn off run_post (RUN_TASK_RUN_POST=FALSE) in setup.sh +# +#----------------------------------------------------------------------- +# +DT_ATMOS: "" +RESTART_INTERVAL: "0" +WRITE_DOPOST: "FALSE" +# +#----------------------------------------------------------------------- +# +# Set METplus parameters. Definitions: +# +# MODEL: +# String that specifies a descriptive name for the model being verified. +# +# MET_INSTALL_DIR: +# Location to top-level directory of MET installation. +# +# METPLUS_PATH: +# Location to top-level directory of METplus installation. +# +# CCPA_OBS_DIR: +# User-specified location of top-level directory where CCPA hourly +# precipitation files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via +# the get_obs_ccpa_tn task (activated in workflow by setting +# RUN_TASK_GET_OBS_CCPA="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/ccpa/proc. METplus is configured to verify 01-, +# 03-, 06-, and 24-h accumulated precipitation using hourly CCPA files. +# METplus configuration files require the use of predetermined directory +# structure and file names. Therefore, if the CCPA files are user +# provided, they need to follow the anticipated naming structure: +# {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, where YYYY is the 4-digit +# valid year, MM the 2-digit valid month, DD the 2-digit valid day of +# the month, and HH the 2-digit valid hour of the day. In addition, a +# caveat is noted for using hourly CCPA data. There is a problem with +# the valid time in the metadata for files valid from 19 - 00 UTC (or +# files under the '00' directory). The script to pull the CCPA data +# from the NOAA HPSS has an example of how to account for this as well +# as organizing the data into a more intuitive format: +# regional_workflow/scripts/exregional_get_ccpa_files.sh. When a fix +# is provided, it will be accounted for in the +# exregional_get_ccpa_files.sh script. +# +# MRMS_OBS_DIR: +# User-specified location of top-level directory where MRMS composite +# reflectivity files used by METplus are located. This parameter needs +# to be set for both user-provided observations and for observations +# that are retrieved from the NOAA HPSS (if the user has access) via the +# get_obs_mrms_tn task (activated in workflow by setting +# RUN_TASK_GET_OBS_MRMS="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/mrms/proc. METplus configuration files require the +# use of predetermined directory structure and file names. Therefore, if +# the MRMS files are user provided, they need to follow the anticipated +# naming structure: +# {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, +# where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD +# the 2-digit valid day of the month, HH the 2-digit valid hour of the +# day, mm the 2-digit valid minutes of the hour, and SS is the two-digit +# valid seconds of the hour. In addition, METplus is configured to look +# for a MRMS composite reflectivity file for the valid time of the +# forecast being verified; since MRMS composite reflectivity files do +# not always exactly match the valid time, a script, within the main +# script to retrieve MRMS data from the NOAA HPSS, is used to identify +# and rename the MRMS composite reflectivity file to match the valid +# time of the forecast. The script to pull the MRMS data from the NOAA +# HPSS has an example of the expected file naming structure: +# regional_workflow/scripts/exregional_get_mrms_files.sh. This script +# calls the script used to identify the MRMS file closest to the valid +# time: regional_workflow/ush/mrms_pull_topofhour.py. +# +# NDAS_OBS_DIR: +# User-specified location of top-level directory where NDAS prepbufr +# files used by METplus are located. This parameter needs to be set for +# both user-provided observations and for observations that are +# retrieved from the NOAA HPSS (if the user has access) via the +# get_obs_ndas_tn task (activated in workflow by setting  +# RUN_TASK_GET_OBS_NDAS="TRUE"). In the case of pulling observations +# directly from NOAA HPSS, the data retrieved will be placed in this +# directory. Please note, this path must be defind as +# /full-path-to-obs/ndas/proc. METplus is configured to verify +# near-surface variables hourly and upper-air variables at times valid +# at 00 and 12 UTC with NDAS prepbufr files. METplus configuration files +# require the use of predetermined file names. Therefore, if the NDAS +# files are user provided, they need to follow the anticipated naming +# structure: prepbufr.ndas.{YYYYMMDDHH}, where YYYY is the 4-digit valid +# year, MM the 2-digit valid month, DD the 2-digit valid day of the +# month, and HH the 2-digit valid hour of the day. The script to pull +# the NDAS data from the NOAA HPSS has an example of how to rename the +# NDAS data into a more intuitive format with the valid time listed in +# the file name: regional_workflow/scripts/exregional_get_ndas_files.sh +# +#----------------------------------------------------------------------- +# +MODEL: "" +MET_INSTALL_DIR: "" +MET_BIN_EXEC: "bin" +METPLUS_PATH: "" +CCPA_OBS_DIR: "" +MRMS_OBS_DIR: "" +NDAS_OBS_DIR: "" +# +#----------------------------------------------------------------------- +# +# Set initial and lateral boundary condition generation parameters. +# Definitions: +# +# EXTRN_MDL_NAME_ICS: +#`The name of the external model that will provide fields from which +# initial condition (including and surface) files will be generated for +# input into the forecast model. +# +# EXTRN_MDL_NAME_LBCS: +#`The name of the external model that will provide fields from which +# lateral boundary condition (LBC) files will be generated for input into +# the forecast model. +# +# LBC_SPEC_INTVL_HRS: +# The interval (in integer hours) with which LBC files will be generated. +# We will refer to this as the boundary update interval. Note that the +# model specified in EXTRN_MDL_NAME_LBCS must have data available at a +# frequency greater than or equal to that implied by LBC_SPEC_INTVL_HRS. +# For example, if LBC_SPEC_INTVL_HRS is set to 6, then the model must have +# data availble at least every 6 hours. It is up to the user to ensure +# that this is the case. +# +# EXTRN_MDL_ICS_OFFSET_HRS: +# Users may wish to start a forecast from a forecast of a previous cycle +# of an external model. This variable sets the number of hours earlier +# the external model started than when the FV3 forecast configured here +# should start. For example, the forecast should start from a 6 hour +# forecast of the GFS, then EXTRN_MDL_ICS_OFFSET_HRS=6. + +# EXTRN_MDL_LBCS_OFFSET_HRS: +# Users may wish to use lateral boundary conditions from a forecast that +# was started earlier than the initial time for the FV3 forecast +# configured here. This variable sets the number of hours earlier +# the external model started than when the FV3 forecast configured here +# should start. For example, the forecast should use lateral boundary +# conditions from the GFS started 6 hours earlier, then +# EXTRN_MDL_LBCS_OFFSET_HRS=6. +# Note: the default value is model-dependent and set in +# set_extrn_mdl_params.sh +# +# FV3GFS_FILE_FMT_ICS: +# If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS +# is set to "FV3GFS"), this variable specifies the format of the model +# files to use when generating the ICs. +# +# FV3GFS_FILE_FMT_LBCS: +# If using the FV3GFS model as the source of the LBCs (i.e. if +# EXTRN_MDL_NAME_LBCS is set to "FV3GFS"), this variable specifies the +# format of the model files to use when generating the LBCs. +# +#----------------------------------------------------------------------- +# +EXTRN_MDL_NAME_ICS: "FV3GFS" +EXTRN_MDL_NAME_LBCS: "FV3GFS" +LBC_SPEC_INTVL_HRS: "6" +EXTRN_MDL_ICS_OFFSET_HRS: "0" +EXTRN_MDL_LBCS_OFFSET_HRS: "" +FV3GFS_FILE_FMT_ICS: "nemsio" +FV3GFS_FILE_FMT_LBCS: "nemsio" +# +#----------------------------------------------------------------------- +# +# Base directories in which to search for external model files. +# +# EXTRN_MDL_SYSBASEDIR_ICS: +# Base directory on the local machine containing external model files for +# generating ICs on the native grid. The way the full path containing +# these files is constructed depends on the user-specified external model +# for ICs, i.e. EXTRN_MDL_NAME_ICS. +# +# EXTRN_MDL_SYSBASEDIR_LBCS: +# Same as EXTRN_MDL_SYSBASEDIR_ICS but for LBCs. +# +# Note that these must be defined as null strings here so that if they +# are specified by the user in the experiment configuration file, they +# remain set to those values, and if not, they get set to machine-dependent +# values. +# +#----------------------------------------------------------------------- +# +EXTRN_MDL_SYSBASEDIR_ICS: '' +EXTRN_MDL_SYSBASEDIR_LBCS: '' +# +#----------------------------------------------------------------------- +# +# User-staged external model directories and files. Definitions: +# +# USE_USER_STAGED_EXTRN_FILES: +# Flag that determines whether or not the workflow will look for the +# external model files needed for generating ICs and LBCs in user-specified +# directories. +# +# EXTRN_MDL_SOURCE_BASEDIR_ICS: +# Directory in which to look for external model files for generating ICs. +# If USE_USER_STAGED_EXTRN_FILES is set to "TRUE", the workflow looks in +# this directory (specifically, in a subdirectory under this directory +# named "YYYYMMDDHH" consisting of the starting date and cycle hour of +# the forecast, where YYYY is the 4-digit year, MM the 2-digit month, DD +# the 2-digit day of the month, and HH the 2-digit hour of the day) for +# the external model files specified by the array EXTRN_MDL_FILES_ICS +# (these files will be used to generate the ICs on the native FV3-LAM +# grid). This variable is not used if USE_USER_STAGED_EXTRN_FILES is +# set to "FALSE". +# +# EXTRN_MDL_FILES_ICS: +# Array containing templates of the names of the files to search for in +# the directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This +# variable is not used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# A single template should be used for each model file type that is +# meant to be used. You may use any of the Python-style templates +# allowed in the ush/retrieve_data.py script. To see the full list of +# supported templates, run that script with a -h option. Here is an example of +# setting FV3GFS nemsio input files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio \ +# gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio ) +# Or for FV3GFS grib files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} ) +# +# EXTRN_MDL_SOURCE_BASEDIR_LBCS: +# Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. +# +# EXTRN_MDL_FILES_LBCS: +# Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. +# +#----------------------------------------------------------------------- +# +USE_USER_STAGED_EXTRN_FILES: "FALSE" +EXTRN_MDL_SOURCE_BASEDIR_ICS: "" +EXTRN_MDL_FILES_ICS: "" +EXTRN_MDL_SOURCE_BASEDIR_LBCS: "" +EXTRN_MDL_FILES_LBCS: "" +# +#----------------------------------------------------------------------- +# +# Set NOMADS online data associated parameters. Definitions: +# +# NOMADS: +# Flag controlling whether or not using NOMADS online data. +# +# NOMADS_file_type: +# Flag controlling the format of data. +# +#----------------------------------------------------------------------- +# +NOMADS: "FALSE" +NOMADS_file_type: "nemsio" +# +#----------------------------------------------------------------------- +# +# Set CCPP-associated parameters. Definitions: +# +# CCPP_PHYS_SUITE: +# The physics suite that will run using CCPP (Common Community Physics +# Package). The choice of physics suite determines the forecast model's +# namelist file, the diagnostics table file, the field table file, and +# the XML physics suite definition file that are staged in the experiment +# directory or the cycle directories under it. +# +#----------------------------------------------------------------------- +# +CCPP_PHYS_SUITE: "FV3_GFS_v16" +# +#----------------------------------------------------------------------- +# +# Set GRID_GEN_METHOD. This variable specifies the method to use to +# generate a regional grid in the horizontal. The values that it can +# take on are: +# +# * "GFDLgrid": +# This setting will generate a regional grid by first generating a +# "parent" global cubed-sphere grid and then taking a portion of tile +# 6 of that global grid -- referred to in the grid generation scripts +# as "tile 7" even though it doesn't correspond to a complete tile -- +# and using it as the regional grid. Note that the forecast is run on +# only on the regional grid (i.e. tile 7, not tiles 1 through 6). +# +# * "ESGgrid": +# This will generate a regional grid using the map projection developed +# by Jim Purser of EMC. +# +# Note that: +# +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then GRID_GEN_METHOD will be reset to the value of +# GRID_GEN_METHOD for that grid. This will happen regardless of +# whether or not GRID_GEN_METHOD is assigned a value in the user- +# specified experiment configuration file, i.e. any value it may be +# assigned in the experiment configuration file will be overwritten. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then GRID_GEN_METHOD must +# be set in the experiment configuration file. Otherwise, it will +# remain set to a null string, and the experiment generation will +# fail because the generation scripts check to ensure that it is set +# to a non-empty string before creating the experiment directory. +# +#----------------------------------------------------------------------- +# +GRID_GEN_METHOD: "" +# +#----------------------------------------------------------------------- +# +# Set parameters specific to the "GFDLgrid" method of generating a regional +# grid (i.e. for GRID_GEN_METHOD set to "GFDLgrid"). The following +# parameters will be used only if GRID_GEN_METHOD is set to "GFDLgrid". +# In this grid generation method: +# +# * The regional grid is defined with respect to a "parent" global cubed- +# sphere grid. Thus, all the parameters for a global cubed-sphere grid +# must be specified in order to define this parent global grid even +# though the model equations are not integrated on (they are integrated +# only on the regional grid). +# +# * GFDLgrid_RES is the number of grid cells in either one of the two +# horizontal directions x and y on any one of the 6 tiles of the parent +# global cubed-sphere grid. The mapping from GFDLgrid_RES to a nominal +# resolution (grid cell size) for a uniform global grid (i.e. Schmidt +# stretch factor GFDLgrid_STRETCH_FAC set to 1) for several values of +# GFDLgrid_RES is as follows: +# +# GFDLgrid_RES typical cell size +# ------------ ----------------- +# 192 50 km +# 384 25 km +# 768 13 km +# 1152 8.5 km +# 3072 3.2 km +# +# Note that these are only typical cell sizes. The actual cell size on +# the global grid tiles varies somewhat as we move across a tile. +# +# * Tile 6 has arbitrarily been chosen as the tile to use to orient the +# global parent grid on the sphere (Earth). This is done by specifying +# GFDLgrid_LON_T6_CTR and GFDLgrid_LAT_T6_CTR, which are the longitude +# and latitude (in degrees) of the center of tile 6. +# +# * Setting the Schmidt stretching factor GFDLgrid_STRETCH_FAC to a value +# greater than 1 shrinks tile 6, while setting it to a value less than +# 1 (but still greater than 0) expands it. The remaining 5 tiles change +# shape as necessary to maintain global coverage of the grid. +# +# * The cell size on a given global tile depends on both GFDLgrid_RES and +# GFDLgrid_STRETCH_FAC (since changing GFDLgrid_RES changes the number +# of cells in the tile, and changing GFDLgrid_STRETCH_FAC modifies the +# shape and size of the tile). +# +# * The regional grid is embedded within tile 6 (i.e. it doesn't extend +# beyond the boundary of tile 6). Its exact location within tile 6 is +# is determined by specifying the starting and ending i and j indices +# of the regional grid on tile 6, where i is the grid index in the x +# direction and j is the grid index in the y direction. These indices +# are stored in the variables +# +# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G +# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G +# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G +# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G +# +# * In the forecast model code and in the experiment generation and workflow +# scripts, for convenience the regional grid is denoted as "tile 7" even +# though it doesn't map back to one of the 6 faces of the cube from +# which the parent global grid is generated (it maps back to only a +# subregion on face 6 since it is wholly confined within tile 6). Tile +# 6 may be referred to as the "parent" tile of the regional grid. +# +# * GFDLgrid_REFINE_RATIO is the refinement ratio of the regional grid +# (tile 7) with respect to the grid on its parent tile (tile 6), i.e. +# it is the number of grid cells along the boundary of the regional grid +# that abut one cell on tile 6. Thus, the cell size on the regional +# grid depends not only on GFDLgrid_RES and GFDLgrid_STRETCH_FAC (because +# the cell size on tile 6 depends on these two parameters) but also on +# GFDLgrid_REFINE_RATIO. Note that as on the tiles of the global grid, +# the cell size on the regional grid is not uniform but varies as we +# move across the grid. +# +# Definitions of parameters that need to be specified when GRID_GEN_METHOD +# is set to "GFDLgrid": +# +# GFDLgrid_LON_T6_CTR: +# Longitude of the center of tile 6 (in degrees). +# +# GFDLgrid_LAT_T6_CTR: +# Latitude of the center of tile 6 (in degrees). +# +# GFDLgrid_RES: +# Number of points in each of the two horizontal directions (x and y) on +# each tile of the parent global grid. Note that the name of this parameter +# is really a misnomer because although it has the stirng "RES" (for +# "resolution") in its name, it specifies number of grid cells, not grid +# size (in say meters or kilometers). However, we keep this name in order +# to remain consistent with the usage of the word "resolution" in the +# global forecast model and other auxiliary codes. +# +# GFDLgrid_STRETCH_FAC: +# Stretching factor used in the Schmidt transformation applied to the +# parent cubed-sphere grid. +# +# GFDLgrid_REFINE_RATIO: +# Cell refinement ratio for the regional grid, i.e. the number of cells +# in either the x or y direction on the regional grid (tile 7) that abut +# one cell on its parent tile (tile 6). +# +# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: +# i-index on tile 6 at which the regional grid (tile 7) starts. +# +# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: +# i-index on tile 6 at which the regional grid (tile 7) ends. +# +# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: +# j-index on tile 6 at which the regional grid (tile 7) starts. +# +# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: +# j-index on tile 6 at which the regional grid (tile 7) ends. +# +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: +# Flag that determines the file naming convention to use for grid, orography, +# and surface climatology files (or, if using pregenerated files, the +# naming convention that was used to name these files). These files +# usually start with the string "C${RES}_", where RES is an integer. +# In the global forecast model, RES is the number of points in each of +# the two horizontal directions (x and y) on each tile of the global grid +# (defined here as GFDLgrid_RES). If this flag is set to "TRUE", RES will +# be set to GFDLgrid_RES just as in the global forecast model. If it is +# set to "FALSE", we calculate (in the grid generation task) an "equivalent +# global uniform cubed-sphere resolution" -- call it RES_EQUIV -- and +# then set RES equal to it. RES_EQUIV is the number of grid points in +# each of the x and y directions on each tile that a global UNIFORM (i.e. +# stretch factor of 1) cubed-sphere grid would have to have in order to +# have the same average grid size as the regional grid. This is a more +# useful indicator of the grid size because it takes into account the +# effects of GFDLgrid_RES, GFDLgrid_STRETCH_FAC, and GFDLgrid_REFINE_RATIO +# in determining the regional grid's typical grid size, whereas simply +# setting RES to GFDLgrid_RES doesn't take into account the effects of +# GFDLgrid_STRETCH_FAC and GFDLgrid_REFINE_RATIO on the regional grid's +# resolution. Nevertheless, some users still prefer to use GFDLgrid_RES +# in the file names, so we allow for that here by setting this flag to +# "TRUE". +# +# Note that: +# +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then: +# +# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then +# these parameters will get reset to the values for that grid. +# This will happen regardless of whether or not they are assigned +# values in the user-specified experiment configuration file, i.e. +# any values they may be assigned in the experiment configuration +# file will be overwritten. +# +# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then +# these parameters will not be used and thus do not need to be reset +# to non-empty strings. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then: +# +# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified +# experiment configuration file, then these parameters must be set +# in that configuration file. +# +# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified +# experiment configuration file, then these parameters will not be +# used and thus do not need to be reset to non-empty strings. +# +#----------------------------------------------------------------------- +# +GFDLgrid_LON_T6_CTR: "" +GFDLgrid_LAT_T6_CTR: "" +GFDLgrid_RES: "" +GFDLgrid_STRETCH_FAC: "" +GFDLgrid_REFINE_RATIO: "" +GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: "" +GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: "" +GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: "" +GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: "" +GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: "" +# +#----------------------------------------------------------------------- +# +# Set parameters specific to the "ESGgrid" method of generating a regional +# grid (i.e. for GRID_GEN_METHOD set to "ESGgrid"). Definitions: +# +# ESGgrid_LON_CTR: +# The longitude of the center of the grid (in degrees). +# +# ESGgrid_LAT_CTR: +# The latitude of the center of the grid (in degrees). +# +# ESGgrid_DELX: +# The cell size in the zonal direction of the regional grid (in meters). +# +# ESGgrid_DELY: +# The cell size in the meridional direction of the regional grid (in +# meters). +# +# ESGgrid_NX: +# The number of cells in the zonal direction on the regional grid. +# +# ESGgrid_NY: +# The number of cells in the meridional direction on the regional grid. +# +# ESGgrid_WIDE_HALO_WIDTH: +# The width (in units of number of grid cells) of the halo to add around +# the regional grid before shaving the halo down to the width(s) expected +# by the forecast model. +# +# ESGgrid_PAZI: +# The rotational parameter for the ESG grid (in degrees). +# +# In order to generate grid files containing halos that are 3-cell and +# 4-cell wide and orography files with halos that are 0-cell and 3-cell +# wide (all of which are required as inputs to the forecast model), the +# grid and orography tasks first create files with halos around the regional +# domain of width ESGgrid_WIDE_HALO_WIDTH cells. These are first stored +# in files. The files are then read in and "shaved" down to obtain grid +# files with 3-cell-wide and 4-cell-wide halos and orography files with +# 0-cell-wide (i.e. no halo) and 3-cell-wide halos. For this reason, we +# refer to the original halo that then gets shaved down as the "wide" +# halo, i.e. because it is wider than the 0-cell-wide, 3-cell-wide, and +# 4-cell-wide halos that we will eventually end up with. Note that the +# grid and orography files with the wide halo are only needed as intermediates +# in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; +# they are not needed by the forecast model. +# NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified +# variable. Just set it in the function set_gridparams_ESGgrid.sh. +# +# Note that: +# +# 1) If the experiment is using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to the name of one of the valid predefined +# grids), then: +# +# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then +# these parameters will not be used and thus do not need to be reset +# to non-empty strings. +# +# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then +# these parameters will get reset to the values for that grid. +# This will happen regardless of whether or not they are assigned +# values in the user-specified experiment configuration file, i.e. +# any values they may be assigned in the experiment configuration +# file will be overwritten. +# +# 2) If the experiment is not using one of the predefined grids (i.e. if +# PREDEF_GRID_NAME is set to a null string), then: +# +# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified +# experiment configuration file, then these parameters will not be +# used and thus do not need to be reset to non-empty strings. +# +# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified +# experiment configuration file, then these parameters must be set +# in that configuration file. +# +#----------------------------------------------------------------------- +# +ESGgrid_LON_CTR: "" +ESGgrid_LAT_CTR: "" +ESGgrid_DELX: "" +ESGgrid_DELY: "" +ESGgrid_NX: "" +ESGgrid_NY: "" +ESGgrid_WIDE_HALO_WIDTH: "" +ESGgrid_PAZI: "" +# +#----------------------------------------------------------------------- +# +# Set computational parameters for the forecast. Definitions: +# +# LAYOUT_X, LAYOUT_Y: +# The number of MPI tasks (processes) to use in the two horizontal +# directions (x and y) of the regional grid when running the forecast +# model. +# +# BLOCKSIZE: +# The amount of data that is passed into the cache at a time. +# +# Here, we set these parameters to null strings. This is so that, for +# any one of these parameters: +# +# 1) If the experiment is using a predefined grid, then if the user +# sets the parameter in the user-specified experiment configuration +# file (EXPT_CONFIG_FN), that value will be used in the forecast(s). +# Otherwise, the default value of the parameter for that predefined +# grid will be used. +# +# 2) If the experiment is not using a predefined grid (i.e. it is using +# a custom grid whose parameters are specified in the experiment +# configuration file), then the user must specify a value for the +# parameter in that configuration file. Otherwise, the parameter +# will remain set to a null string, and the experiment generation +# will fail because the generation scripts check to ensure that all +# the parameters defined in this section are set to non-empty strings +# before creating the experiment directory. +# +#----------------------------------------------------------------------- +# +LAYOUT_X: "" +LAYOUT_Y: "" +BLOCKSIZE: "" +# +#----------------------------------------------------------------------- +# +# Set write-component (quilting) parameters. Definitions: +# +# QUILTING: +# Flag that determines whether or not to use the write component for +# writing output files to disk. +# +# WRTCMP_write_groups: +# The number of write groups (i.e. groups of MPI tasks) to use in the +# write component. +# +# WRTCMP_write_tasks_per_group: +# The number of MPI tasks to allocate for each write group. +# +# PRINT_ESMF: +# Flag for whether or not to output extra (debugging) information from +# ESMF routines. Must be "TRUE" or "FALSE". Note that the write +# component uses ESMF library routines to interpolate from the native +# forecast model grid to the user-specified output grid (which is defined +# in the model configuration file MODEL_CONFIG_FN in the forecast's run +# directory). +# +#----------------------------------------------------------------------- +# +QUILTING: "TRUE" +PRINT_ESMF: "FALSE" + +WRTCMP_write_groups: "1" +WRTCMP_write_tasks_per_group: "20" + +WRTCMP_output_grid: "''" +WRTCMP_cen_lon: "" +WRTCMP_cen_lat: "" +WRTCMP_lon_lwr_left: "" +WRTCMP_lat_lwr_left: "" +# +# The following are used only for the case of WRTCMP_output_grid set to +# "'rotated_latlon'". +# +WRTCMP_lon_upr_rght: "" +WRTCMP_lat_upr_rght: "" +WRTCMP_dlon: "" +WRTCMP_dlat: "" +# +# The following are used only for the case of WRTCMP_output_grid set to +# "'lambert_conformal'". +# +WRTCMP_stdlat1: "" +WRTCMP_stdlat2: "" +WRTCMP_nx: "" +WRTCMP_ny: "" +WRTCMP_dx: "" +WRTCMP_dy: "" +# +#----------------------------------------------------------------------- +# +# Set PREDEF_GRID_NAME. This parameter specifies a predefined regional +# grid, as follows: +# +# * If PREDEF_GRID_NAME is set to a valid predefined grid name, the grid +# generation method GRID_GEN_METHOD, the (native) grid parameters, and +# the write-component grid parameters are set to predefined values for +# the specified grid, overwriting any settings of these parameters in +# the user-specified experiment configuration file. In addition, if +# the time step DT_ATMOS and the computational parameters LAYOUT_X, +# LAYOUT_Y, and BLOCKSIZE are not specified in that configuration file, +# they are also set to predefined values for the specified grid. +# +# * If PREDEF_GRID_NAME is set to an empty string, it implies the user +# is providing the native grid parameters in the user-specified +# experiment configuration file (EXPT_CONFIG_FN). In this case, the +# grid generation method GRID_GEN_METHOD, the native grid parameters, +# and the write-component grid parameters as well as the time step +# forecast model's main time step DT_ATMOS and the computational +# parameters LAYOUT_X, LAYOUT_Y, and BLOCKSIZE must be set in that +# configuration file; otherwise, the values of all of these parameters +# in this default experiment configuration file will be used. +# +# Setting PREDEF_GRID_NAME provides a convenient method of specifying a +# commonly used set of grid-dependent parameters. The predefined grid +# parameters are specified in the script +# +# $HOMErrfs/ush/set_predef_grid_params.sh +# +#----------------------------------------------------------------------- +# +PREDEF_GRID_NAME: "" +# +#----------------------------------------------------------------------- +# +# Set PREEXISTING_DIR_METHOD. This variable determines the method to use +# use to deal with preexisting directories [e.g ones generated by previous +# calls to the experiment generation script using the same experiment name +# (EXPT_SUBDIR) as the current experiment]. This variable must be set to +# one of "delete", "rename", and "quit". The resulting behavior for each +# of these values is as follows: +# +# * "delete": +# The preexisting directory is deleted and a new directory (having the +# same name as the original preexisting directory) is created. +# +# * "rename": +# The preexisting directory is renamed and a new directory (having the +# same name as the original preexisting directory) is created. The new +# name of the preexisting directory consists of its original name and +# the suffix "_oldNNN", where NNN is a 3-digit integer chosen to make +# the new name unique. +# +# * "quit": +# The preexisting directory is left unchanged, but execution of the +# currently running script is terminated. In this case, the preexisting +# directory must be dealt with manually before rerunning the script. +# +#----------------------------------------------------------------------- +# +PREEXISTING_DIR_METHOD: "delete" +# +#----------------------------------------------------------------------- +# +# Set flags for more detailed messages. Defintitions: +# +# VERBOSE: +# This is a flag that determines whether or not the experiment generation +# and workflow task scripts tend to print out more informational messages. +# +# DEBUG: +# This is a flag that determines whether or not very detailed debugging +# messages are printed to out. Note that if DEBUG is set to TRUE, then +# VERBOSE will also get reset to TRUE if it isn't already. +# +#----------------------------------------------------------------------- +# +VERBOSE: "TRUE" +DEBUG: "FALSE" +# +#----------------------------------------------------------------------- +# +# Set the names of the various rocoto workflow tasks. +# +#----------------------------------------------------------------------- +# +MAKE_GRID_TN: "make_grid" +MAKE_OROG_TN: "make_orog" +MAKE_SFC_CLIMO_TN: "make_sfc_climo" +GET_EXTRN_ICS_TN: "get_extrn_ics" +GET_EXTRN_LBCS_TN: "get_extrn_lbcs" +MAKE_ICS_TN: "make_ics" +MAKE_LBCS_TN: "make_lbcs" +RUN_FCST_TN: "run_fcst" +RUN_POST_TN: "run_post" +GET_OBS: "get_obs" +GET_OBS_CCPA_TN: "get_obs_ccpa" +GET_OBS_MRMS_TN: "get_obs_mrms" +GET_OBS_NDAS_TN: "get_obs_ndas" +VX_TN: "run_vx" +VX_GRIDSTAT_TN: "run_gridstatvx" +VX_GRIDSTAT_REFC_TN: "run_gridstatvx_refc" +VX_GRIDSTAT_RETOP_TN: "run_gridstatvx_retop" +VX_GRIDSTAT_03h_TN: "run_gridstatvx_03h" +VX_GRIDSTAT_06h_TN: "run_gridstatvx_06h" +VX_GRIDSTAT_24h_TN: "run_gridstatvx_24h" +VX_POINTSTAT_TN: "run_pointstatvx" +VX_ENSGRID_TN: "run_ensgridvx" +VX_ENSGRID_03h_TN: "run_ensgridvx_03h" +VX_ENSGRID_06h_TN: "run_ensgridvx_06h" +VX_ENSGRID_24h_TN: "run_ensgridvx_24h" +VX_ENSGRID_REFC_TN: "run_ensgridvx_refc" +VX_ENSGRID_RETOP_TN: "run_ensgridvx_retop" +VX_ENSGRID_MEAN_TN: "run_ensgridvx_mean" +VX_ENSGRID_PROB_TN: "run_ensgridvx_prob" +VX_ENSGRID_MEAN_03h_TN: "run_ensgridvx_mean_03h" +VX_ENSGRID_PROB_03h_TN: "run_ensgridvx_prob_03h" +VX_ENSGRID_MEAN_06h_TN: "run_ensgridvx_mean_06h" +VX_ENSGRID_PROB_06h_TN: "run_ensgridvx_prob_06h" +VX_ENSGRID_MEAN_24h_TN: "run_ensgridvx_mean_24h" +VX_ENSGRID_PROB_24h_TN: "run_ensgridvx_prob_24h" +VX_ENSGRID_PROB_REFC_TN: "run_ensgridvx_prob_refc" +VX_ENSGRID_PROB_RETOP_TN: "run_ensgridvx_prob_retop" +VX_ENSPOINT_TN: "run_enspointvx" +VX_ENSPOINT_MEAN_TN: "run_enspointvx_mean" +VX_ENSPOINT_PROB_TN: "run_enspointvx_prob" +# +#----------------------------------------------------------------------- +# +# Set flags (and related directories) that determine whether various +# workflow tasks should be run. Note that the MAKE_GRID_TN, MAKE_OROG_TN, +# and MAKE_SFC_CLIMO_TN are all cycle-independent tasks, i.e. if they +# are to be run, they do so only once at the beginning of the workflow +# before any cycles are run. Definitions: +# +# RUN_TASK_MAKE_GRID: +# Flag that determines whether the MAKE_GRID_TN task is to be run. If +# this is set to "TRUE", the grid generation task is run and new grid +# files are generated. If it is set to "FALSE", then the scripts look +# for pregenerated grid files in the directory specified by GRID_DIR +# (see below). +# +# GRID_DIR: +# The directory in which to look for pregenerated grid files if +# RUN_TASK_MAKE_GRID is set to "FALSE". +# +# RUN_TASK_MAKE_OROG: +# Same as RUN_TASK_MAKE_GRID but for the MAKE_OROG_TN task. +# +# OROG_DIR: +# Same as GRID_DIR but for the MAKE_OROG_TN task. +# +# RUN_TASK_MAKE_SFC_CLIMO: +# Same as RUN_TASK_MAKE_GRID but for the MAKE_SFC_CLIMO_TN task. +# +# SFC_CLIMO_DIR: +# Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. +# +# RUN_TASK_GET_EXTRN_ICS: +# Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. +# +# RUN_TASK_GET_EXTRN_LBCS: +# Flag that determines whether the GET_EXTRN_LBCS_TN task is to be run. +# +# RUN_TASK_MAKE_ICS: +# Flag that determines whether the MAKE_ICS_TN task is to be run. +# +# RUN_TASK_MAKE_LBCS: +# Flag that determines whether the MAKE_LBCS_TN task is to be run. +# +# RUN_TASK_RUN_FCST: +# Flag that determines whether the RUN_FCST_TN task is to be run. +# +# RUN_TASK_RUN_POST: +# Flag that determines whether the RUN_POST_TN task is to be run. +# +# RUN_TASK_VX_GRIDSTAT: +# Flag that determines whether the grid-stat verification task is to be +# run. +# +# RUN_TASK_VX_POINTSTAT: +# Flag that determines whether the point-stat verification task is to be +# run. +# +# RUN_TASK_VX_ENSGRID: +# Flag that determines whether the ensemble-stat verification for gridded +# data task is to be run. +# +# RUN_TASK_VX_ENSPOINT: +# Flag that determines whether the ensemble point verification task is +# to be run. If this flag is set, both ensemble-stat point verification +# and point verification of ensemble-stat output is computed. +# +#----------------------------------------------------------------------- +# +RUN_TASK_MAKE_GRID: "TRUE" +GRID_DIR: "/path/to/pregenerated/grid/files" + +RUN_TASK_MAKE_OROG: "TRUE" +OROG_DIR: "/path/to/pregenerated/orog/files" + +RUN_TASK_MAKE_SFC_CLIMO: "TRUE" +SFC_CLIMO_DIR: "/path/to/pregenerated/surface/climo/files" + +RUN_TASK_GET_EXTRN_ICS: "TRUE" +RUN_TASK_GET_EXTRN_LBCS: "TRUE" +RUN_TASK_MAKE_ICS: "TRUE" +RUN_TASK_MAKE_LBCS: "TRUE" +RUN_TASK_RUN_FCST: "TRUE" +RUN_TASK_RUN_POST: "TRUE" + +RUN_TASK_GET_OBS_CCPA: "FALSE" +RUN_TASK_GET_OBS_MRMS: "FALSE" +RUN_TASK_GET_OBS_NDAS: "FALSE" +RUN_TASK_VX_GRIDSTAT: "FALSE" +RUN_TASK_VX_POINTSTAT: "FALSE" +RUN_TASK_VX_ENSGRID: "FALSE" +RUN_TASK_VX_ENSPOINT: "FALSE" +# +#----------------------------------------------------------------------- +# +# Flag that determines whether MERRA2 aerosol climatology data and +# lookup tables for optics properties are obtained +# +#----------------------------------------------------------------------- +# +USE_MERRA_CLIMO: "FALSE" +# +#----------------------------------------------------------------------- +# +# Set the array parameter containing the names of all the fields that the +# MAKE_SFC_CLIMO_TN task generates on the native FV3-LAM grid. +# +#----------------------------------------------------------------------- +# +SFC_CLIMO_FIELDS: [ +"facsf", +"maximum_snow_albedo", +"slope_type", +"snowfree_albedo", +"soil_type", +"substrate_temperature", +"vegetation_greenness", +"vegetation_type" +] +# +#----------------------------------------------------------------------- +# +# Set parameters associated with the fixed (i.e. static) files. Definitions: +# +# FIXgsm: +# System directory in which the majority of fixed (i.e. time-independent) +# files that are needed to run the FV3-LAM model are located +# +# FIXaer: +# System directory where MERRA2 aerosol climatology files are located +# +# FIXlut: +# System directory where the lookup tables for optics properties are located +# +# TOPO_DIR: +# The location on disk of the static input files used by the make_orog +# task (orog.x and shave.x). Can be the same as FIXgsm. +# +# SFC_CLIMO_INPUT_DIR: +# The location on disk of the static surface climatology input fields, used by +# sfc_climo_gen. These files are only used if RUN_TASK_MAKE_SFC_CLIMO=TRUE +# +# FNGLAC, ..., FNMSKH: +# Names of (some of the) global data files that are assumed to exist in +# a system directory specified (this directory is machine-dependent; +# the experiment generation scripts will set it and store it in the +# variable FIXgsm). These file names also appear directly in the forecast +# model's input namelist file. +# +# FIXgsm_FILES_TO_COPY_TO_FIXam: +# If not running in NCO mode, this array contains the names of the files +# to copy from the FIXgsm system directory to the FIXam directory under +# the experiment directory. Note that the last element has a dummy value. +# This last element will get reset by the workflow generation scripts to +# the name of the ozone production/loss file to copy from FIXgsm. The +# name of this file depends on the ozone parameterization being used, +# and that in turn depends on the CCPP physics suite specified for the +# experiment. Thus, the CCPP physics suite XML must first be read in to +# determine the ozone parameterizaton and then the name of the ozone +# production/loss file. These steps are carried out elsewhere (in one +# of the workflow generation scripts/functions). +# +# FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: +# This array is used to set some of the namelist variables in the forecast +# model's namelist file that represent the relative or absolute paths of +# various fixed files (the first column of the array, where columns are +# delineated by the pipe symbol "|") to the full paths to these files in +# the FIXam directory derived from the corresponding workflow variables +# containing file names (the second column of the array). +# +# FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: +# This array is used to set some of the namelist variables in the forecast +# model's namelist file that represent the relative or absolute paths of +# various fixed files (the first column of the array, where columns are +# delineated by the pipe symbol "|") to the full paths to surface climatology +# files (on the native FV3-LAM grid) in the FIXLAM directory derived from +# the corresponding surface climatology fields (the second column of the +# array). +# +# CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: +# This array specifies the mapping to use between the symlinks that need +# to be created in each cycle directory (these are the "files" that FV3 +# looks for) and their targets in the FIXam directory. The first column +# of the array specifies the symlink to be created, and the second column +# specifies its target file in FIXam (where columns are delineated by the +# pipe symbol "|"). +# +#----------------------------------------------------------------------- +# +# Because the default values are dependent on the platform, we set these +# to a null string which will then be overwritten in setup.sh unless the +# user has specified a different value in config.sh +FIXgsm: "" +FIXaer: "" +FIXlut: "" +TOPO_DIR: "" +SFC_CLIMO_INPUT_DIR: "" + +FNGLAC: &FNGLAC "global_glacier.2x2.grb" +FNMXIC: &FNMXIC "global_maxice.2x2.grb" +FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" +FNSNOC: &FNSNOC "global_snoclim.1.875.grb" +FNZORC: &FNZORC "igbp" +FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" +FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" +FNMSKH: &FNMSKH "seaice_newland.grb" + +FIXgsm_FILES_TO_COPY_TO_FIXam: [ +*FNGLAC, +*FNMXIC, +*FNTSFC, +*FNSNOC, +*FNAISC, +*FNSMCC, +*FNMSKH, +"global_climaeropac_global.txt", +"fix_co2_proj/global_co2historicaldata_2010.txt", +"fix_co2_proj/global_co2historicaldata_2011.txt", +"fix_co2_proj/global_co2historicaldata_2012.txt", +"fix_co2_proj/global_co2historicaldata_2013.txt", +"fix_co2_proj/global_co2historicaldata_2014.txt", +"fix_co2_proj/global_co2historicaldata_2015.txt", +"fix_co2_proj/global_co2historicaldata_2016.txt", +"fix_co2_proj/global_co2historicaldata_2017.txt", +"fix_co2_proj/global_co2historicaldata_2018.txt", +"fix_co2_proj/global_co2historicaldata_2019.txt", +"fix_co2_proj/global_co2historicaldata_2020.txt", +"fix_co2_proj/global_co2historicaldata_2021.txt", +"global_co2historicaldata_glob.txt", +"co2monthlycyc.txt", +"global_h2o_pltc.f77", +"global_hyblev.l65.txt", +"global_zorclim.1x1.grb", +"global_sfc_emissivity_idx.txt", +"global_tg3clim.2.6x1.5.grb", +"global_solarconstant_noaa_an.txt", +"global_albedo4.1x1.grb", +"geo_em.d01.lat-lon.2.5m.HGT_M.nc", +"HGT.Beljaars_filtered.lat-lon.30s_res.nc", +"replace_with_FIXgsm_ozone_prodloss_filename" +] + +# +# It is possible to remove this as a workflow variable and make it only +# a local one since it is used in only one script. +# +FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ +!join_str ["FNGLAC | ",*FNGLAC], +!join_str ["FNMXIC | ",*FNMXIC], +!join_str ["FNTSFC | ",*FNTSFC], +!join_str ["FNSNOC | ",*FNSNOC], +!join_str ["FNAISC | ",*FNAISC], +!join_str ["FNSMCC | ",*FNSMCC], +!join_str ["FNMSKH | ",*FNMSKH] +] +#"FNZORC | $FNZORC", + +FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ +"FNALBC | snowfree_albedo", +"FNALBC2 | facsf", +"FNTG3C | substrate_temperature", +"FNVEGC | vegetation_greenness", +"FNVETC | vegetation_type", +"FNSOTC | soil_type", +"FNVMNC | vegetation_greenness", +"FNVMXC | vegetation_greenness", +"FNSLPC | slope_type", +"FNABSC | maximum_snow_albedo" +] + +CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ +"aerosol.dat | global_climaeropac_global.txt", +"co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", +"co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", +"co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", +"co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", +"co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", +"co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", +"co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", +"co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", +"co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", +"co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", +"co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", +"co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", +"co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", +"co2monthlycyc.txt | co2monthlycyc.txt", +"global_h2oprdlos.f77 | global_h2o_pltc.f77", +"global_albedo4.1x1.grb | global_albedo4.1x1.grb", +"global_zorclim.1x1.grb | global_zorclim.1x1.grb", +"global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", +"sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", +"solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", +"global_o3prdlos.f77 | " +] +# +#----------------------------------------------------------------------- +# +# For each workflow task, set the parameters to pass to the job scheduler +# (e.g. slurm) that will submit a job for each task to be run. These +# parameters include the number of nodes to use to run the job, the MPI +# processes per node, the maximum walltime to allow for the job to complete, +# and the maximum number of times to attempt to run each task. +# +#----------------------------------------------------------------------- +# +# Number of nodes. +# +NNODES_MAKE_GRID: "1" +NNODES_MAKE_OROG: "1" +NNODES_MAKE_SFC_CLIMO: "2" +NNODES_GET_EXTRN_ICS: "1" +NNODES_GET_EXTRN_LBCS: "1" +NNODES_MAKE_ICS: "4" +NNODES_MAKE_LBCS: "4" +NNODES_RUN_FCST: "" # This is calculated in the workflow generation scripts, so no need to set here. +NNODES_RUN_POST: "2" +NNODES_GET_OBS_CCPA: "1" +NNODES_GET_OBS_MRMS: "1" +NNODES_GET_OBS_NDAS: "1" +NNODES_VX_GRIDSTAT: "1" +NNODES_VX_POINTSTAT: "1" +NNODES_VX_ENSGRID: "1" +NNODES_VX_ENSGRID_MEAN: "1" +NNODES_VX_ENSGRID_PROB: "1" +NNODES_VX_ENSPOINT: "1" +NNODES_VX_ENSPOINT_MEAN: "1" +NNODES_VX_ENSPOINT_PROB: "1" +# +# Number of MPI processes per node. +# +PPN_MAKE_GRID: "24" +PPN_MAKE_OROG: "24" +PPN_MAKE_SFC_CLIMO: "24" +PPN_GET_EXTRN_ICS: "1" +PPN_GET_EXTRN_LBCS: "1" +PPN_MAKE_ICS: "12" +PPN_MAKE_LBCS: "12" +PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.sh +PPN_RUN_POST: "24" +PPN_GET_OBS_CCPA: "1" +PPN_GET_OBS_MRMS: "1" +PPN_GET_OBS_NDAS: "1" +PPN_VX_GRIDSTAT: "1" +PPN_VX_POINTSTAT: "1" +PPN_VX_ENSGRID: "1" +PPN_VX_ENSGRID_MEAN: "1" +PPN_VX_ENSGRID_PROB: "1" +PPN_VX_ENSPOINT: "1" +PPN_VX_ENSPOINT_MEAN: "1" +PPN_VX_ENSPOINT_PROB: "1" +# +# Walltimes. +# +WTIME_MAKE_GRID: "00:20:00" +WTIME_MAKE_OROG: "01:00:00" +WTIME_MAKE_SFC_CLIMO: "00:20:00" +WTIME_GET_EXTRN_ICS: "00:45:00" +WTIME_GET_EXTRN_LBCS: "00:45:00" +WTIME_MAKE_ICS: "00:30:00" +WTIME_MAKE_LBCS: "00:30:00" +WTIME_RUN_FCST: "04:30:00" +WTIME_RUN_POST: "00:15:00" +WTIME_GET_OBS_CCPA: "00:45:00" +WTIME_GET_OBS_MRMS: "00:45:00" +WTIME_GET_OBS_NDAS: "02:00:00" +WTIME_VX_GRIDSTAT: "02:00:00" +WTIME_VX_POINTSTAT: "01:00:00" +WTIME_VX_ENSGRID: "01:00:00" +WTIME_VX_ENSGRID_MEAN: "01:00:00" +WTIME_VX_ENSGRID_PROB: "01:00:00" +WTIME_VX_ENSPOINT: "01:00:00" +WTIME_VX_ENSPOINT_MEAN: "01:00:00" +WTIME_VX_ENSPOINT_PROB: "01:00:00" +# +# Maximum number of attempts. +# +MAXTRIES_MAKE_GRID: "2" +MAXTRIES_MAKE_OROG: "2" +MAXTRIES_MAKE_SFC_CLIMO: "2" +MAXTRIES_GET_EXTRN_ICS: "1" +MAXTRIES_GET_EXTRN_LBCS: "1" +MAXTRIES_MAKE_ICS: "1" +MAXTRIES_MAKE_LBCS: "1" +MAXTRIES_RUN_FCST: "1" +MAXTRIES_RUN_POST: "2" +MAXTRIES_GET_OBS_CCPA: "1" +MAXTRIES_GET_OBS_MRMS: "1" +MAXTRIES_GET_OBS_NDAS: "1" +MAXTRIES_VX_GRIDSTAT: "1" +MAXTRIES_VX_GRIDSTAT_REFC: "1" +MAXTRIES_VX_GRIDSTAT_RETOP: "1" +MAXTRIES_VX_GRIDSTAT_03h: "1" +MAXTRIES_VX_GRIDSTAT_06h: "1" +MAXTRIES_VX_GRIDSTAT_24h: "1" +MAXTRIES_VX_POINTSTAT: "1" +MAXTRIES_VX_ENSGRID: "1" +MAXTRIES_VX_ENSGRID_REFC: "1" +MAXTRIES_VX_ENSGRID_RETOP: "1" +MAXTRIES_VX_ENSGRID_03h: "1" +MAXTRIES_VX_ENSGRID_06h: "1" +MAXTRIES_VX_ENSGRID_24h: "1" +MAXTRIES_VX_ENSGRID_MEAN: "1" +MAXTRIES_VX_ENSGRID_PROB: "1" +MAXTRIES_VX_ENSGRID_MEAN_03h: "1" +MAXTRIES_VX_ENSGRID_PROB_03h: "1" +MAXTRIES_VX_ENSGRID_MEAN_06h: "1" +MAXTRIES_VX_ENSGRID_PROB_06h: "1" +MAXTRIES_VX_ENSGRID_MEAN_24h: "1" +MAXTRIES_VX_ENSGRID_PROB_24h: "1" +MAXTRIES_VX_ENSGRID_PROB_REFC: "1" +MAXTRIES_VX_ENSGRID_PROB_RETOP: "1" +MAXTRIES_VX_ENSPOINT: "1" +MAXTRIES_VX_ENSPOINT_MEAN: "1" +MAXTRIES_VX_ENSPOINT_PROB: "1" + +# +#----------------------------------------------------------------------- +# +# Allows an extra parameter to be passed to slurm via XML Native +# command +# +SLURM_NATIVE_CMD: "" +# +#----------------------------------------------------------------------- +# +# Set parameters associated with subhourly forecast model output and +# post-processing. +# +# SUB_HOURLY_POST: +# Flag that indicates whether the forecast model will generate output +# files on a sub-hourly time interval (e.g. 10 minutes, 15 minutes, etc). +# This will also cause the post-processor to process these sub-hourly +# files. If ths is set to "TRUE", then DT_SUBHOURLY_POST_MNTS should be +# set to a value between "00" and "59". +# +# DT_SUB_HOURLY_POST_MNTS: +# Time interval in minutes between the forecast model output files. If +# SUB_HOURLY_POST is set to "TRUE", this needs to be set to a two-digit +# integer between "01" and "59". This is not used if SUB_HOURLY_POST is +# not set to "TRUE". Note that if SUB_HOURLY_POST is set to "TRUE" but +# DT_SUB_HOURLY_POST_MNTS is set to "00", SUB_HOURLY_POST will get reset +# to "FALSE" in the experiment generation scripts (there will be an +# informational message in the log file to emphasize this). +# +#----------------------------------------------------------------------- +# +SUB_HOURLY_POST: "FALSE" +DT_SUBHOURLY_POST_MNTS: "00" +# +#----------------------------------------------------------------------- +# +# Set parameters associated with defining a customized post configuration +# file. +# +# USE_CUSTOM_POST_CONFIG_FILE: +# Flag that determines whether a user-provided custom configuration file +# should be used for post-processing the model data. If this is set to +# "TRUE", then the workflow will use the custom post-processing (UPP) +# configuration file specified in CUSTOM_POST_CONFIG_FP. Otherwise, a +# default configuration file provided in the UPP repository will be +# used. +# +# CUSTOM_POST_CONFIG_FP: +# The full path to the custom post flat file, including filename, to be +# used for post-processing. This is only used if CUSTOM_POST_CONFIG_FILE +# is set to "TRUE". +# +#----------------------------------------------------------------------- +# +USE_CUSTOM_POST_CONFIG_FILE: "FALSE" +CUSTOM_POST_CONFIG_FP: "" +# +#----------------------------------------------------------------------- +# +# Set parameters associated with outputting satellite fields in the UPP +# grib2 files using the Community Radiative Transfer Model (CRTM). +# +# USE_CRTM: +# Flag that defines whether external CRTM coefficient files have been +# staged by the user in order to output synthetic statellite products +# available within the UPP. If this is set to "TRUE", then the workflow +# will check for these files in the directory CRTM_DIR. Otherwise, it is +# assumed that no satellite fields are being requested in the UPP +# configuration. +# +# CRTM_DIR: +# This is the path to the top CRTM fix file directory. This is only used +# if USE_CRTM is set to "TRUE". +# +#----------------------------------------------------------------------- +# +USE_CRTM: "FALSE" +CRTM_DIR: "" +# +#----------------------------------------------------------------------- +# +# Set parameters associated with running ensembles. Definitions: +# +# DO_ENSEMBLE: +# Flag that determines whether to run a set of ensemble forecasts (for +# each set of specified cycles). If this is set to "TRUE", NUM_ENS_MEMBERS +# forecasts are run for each cycle, each with a different set of stochastic +# seed values. Otherwise, a single forecast is run for each cycle. +# +# NUM_ENS_MEMBERS: +# The number of ensemble members to run if DO_ENSEMBLE is set to "TRUE". +# This variable also controls the naming of the ensemble member directories. +# For example, if this is set to "8", the member directories will be named +# mem1, mem2, ..., mem8. If it is set to "08" (note the leading zero), +# the member directories will be named mem01, mem02, ..., mem08. Note, +# however, that after reading in the number of characters in this string +# (in order to determine how many leading zeros, if any, should be placed +# in the names of the member directories), the workflow generation scripts +# strip away those leading zeros. Thus, in the variable definitions file +# (GLOBAL_VAR_DEFNS_FN), this variable appear with its leading zeros +# stripped. This variable is not used if DO_ENSEMBLE is not set to "TRUE". +# +#----------------------------------------------------------------------- +# +DO_ENSEMBLE: "FALSE" +NUM_ENS_MEMBERS: "1" +# +#----------------------------------------------------------------------- +# +# Set default ad-hoc stochastic physics options. +# For detailed documentation of these parameters, see: +# https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html +# +#----------------------------------------------------------------------- +# +DO_SHUM: "FALSE" +DO_SPPT: "FALSE" +DO_SKEB: "FALSE" +ISEED_SPPT: "1" +ISEED_SHUM: "2" +ISEED_SKEB: "3" +NEW_LSCALE: "TRUE" +SHUM_MAG: "0.006" #Variable "shum" in input.nml +SHUM_LSCALE: "150000" +SHUM_TSCALE: "21600" #Variable "shum_tau" in input.nml +SHUM_INT: "3600" #Variable "shumint" in input.nml +SPPT_MAG: "0.7" #Variable "sppt" in input.nml +SPPT_LOGIT: "TRUE" +SPPT_LSCALE: "150000" +SPPT_TSCALE: "21600" #Variable "sppt_tau" in input.nml +SPPT_INT: "3600" #Variable "spptint" in input.nml +SPPT_SFCLIMIT: "TRUE" +SKEB_MAG: "0.5" #Variable "skeb" in input.nml +SKEB_LSCALE: "150000" +SKEB_TSCALE: "21600" #Variable "skeb_tau" in input.nml +SKEB_INT: "3600" #Variable "skebint" in input.nml +SKEBNORM: "1" +SKEB_VDOF: "10" +USE_ZMTNBLCK: "FALSE" +# +#----------------------------------------------------------------------- +# +# Set default SPP stochastic physics options. Each SPP option is an array, +# applicable (in order) to the scheme/parameter listed in SPP_VAR_LIST. +# Enter each value of the array in config.sh as shown below without commas +# or single quotes (e.g., SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ). +# Both commas and single quotes will be added by Jinja when creating the +# namelist. +# +# Note that SPP is currently only available for specific physics schemes +# used in the RAP/HRRR physics suite. Users need to be aware of which SDF +# is chosen when turning this option on. +# +# Patterns evolve and are applied at each time step. +# +#----------------------------------------------------------------------- +# +DO_SPP: "FALSE" +SPP_VAR_LIST: [ "pbl", "sfc", "mp", "rad", "gwd" ] +SPP_MAG_LIST: [ "0.2", "0.2", "0.75", "0.2", "0.2" ] #Variable "spp_prt_list" in input.nml +SPP_LSCALE: [ "150000.0", "150000.0", "150000.0", "150000.0", "150000.0" ] +SPP_TSCALE: [ "21600.0", "21600.0", "21600.0", "21600.0", "21600.0" ] #Variable "spp_tau" in input.nml +SPP_SIGTOP1: [ "0.1", "0.1", "0.1", "0.1", "0.1" ] +SPP_SIGTOP2: [ "0.025", "0.025", "0.025", "0.025", "0.025" ] +SPP_STDDEV_CUTOFF: [ "1.5", "1.5", "2.5", "1.5", "1.5" ] +ISEED_SPP: [ "4", "4", "4", "4", "4" ] +# +#----------------------------------------------------------------------- +# +# Turn on SPP in Noah or RUC LSM (support for Noah MP is in progress). +# Please be aware of the SDF that you choose if you wish to turn on LSM +# SPP. +# +# SPP in LSM schemes is handled in the &nam_sfcperts namelist block +# instead of in &nam_sppperts, where all other SPP is implemented. +# +# The default perturbation frequency is determined by the fhcyc namelist +# entry. Since that parameter is set to zero in the SRW App, use +# LSM_SPP_EACH_STEP to perturb every time step. +# +# Perturbations to soil moisture content (SMC) are only applied at the +# first time step. +# +# LSM perturbations include SMC - soil moisture content (volume fraction), +# VGF - vegetation fraction, ALB - albedo, SAL - salinity, +# EMI - emissivity, ZOL - surface roughness (cm), and STC - soil temperature. +# +# Only five perturbations at a time can be applied currently, but all seven +# are shown below. In addition, only one unique iseed value is allowed +# at the moment, and is used for each pattern. +# +DO_LSM_SPP: "FALSE" #If true, sets lndp_type=2 +LSM_SPP_TSCALE: [ "21600", "21600", "21600", "21600", "21600", "21600", "21600" ] +LSM_SPP_LSCALE: [ "150000", "150000", "150000", "150000", "150000", "150000", "150000" ] +ISEED_LSM_SPP: [ "9" ] +LSM_SPP_VAR_LIST: [ "smc", "vgf", "alb", "sal", "emi", "zol", "stc" ] +LSM_SPP_MAG_LIST: [ "0.2", "0.001", "0.001", "0.001", "0.001", "0.001", "0.2" ] +LSM_SPP_EACH_STEP: "TRUE" #Sets lndp_each_step=.true. +# +#----------------------------------------------------------------------- +# +# HALO_BLEND: +# Number of rows into the computational domain that should be blended +# with the LBCs. To shut halo blending off, this can be set to zero. +# +#----------------------------------------------------------------------- +# +HALO_BLEND: "10" +# +#----------------------------------------------------------------------- +# +# USE_FVCOM: +# Flag set to update surface conditions in FV3-LAM with fields generated +# from the Finite Volume Community Ocean Model (FVCOM). This will +# replace lake/sea surface temperature, ice surface temperature, and ice +# placement. FVCOM data must already be interpolated to the desired +# FV3-LAM grid. This flag will be used in make_ics to modify sfc_data.nc +# after chgres_cube is run by running the routine process_FVCOM.exe +# +# FVCOM_WCSTART: +# Define if this is a "warm" start or a "cold" start. Setting this to +# "warm" will read in sfc_data.nc generated in a RESTART directory. +# Setting this to "cold" will read in the sfc_data.nc generated from +# chgres_cube in the make_ics portion of the workflow. +# +# FVCOM_DIR: +# User defined directory where FVCOM data already interpolated to FV3-LAM +# grid is located. File name in this path should be "fvcom.nc" to allow +# +# FVCOM_FILE: +# Name of file located in FVCOM_DIR that has FVCOM data interpolated to +# FV3-LAM grid. This file will be copied later to a new location and name +# changed to fvcom.nc +# +#------------------------------------------------------------------------ +# +USE_FVCOM: "FALSE" +FVCOM_WCSTART: "cold" +FVCOM_DIR: "/user/defined/dir/to/fvcom/data" +FVCOM_FILE: "fvcom.nc" +# +#----------------------------------------------------------------------- +# +# COMPILER: +# Type of compiler invoked during the build step. +# +#------------------------------------------------------------------------ +# +COMPILER: "intel" +# +#----------------------------------------------------------------------- +# +# KMP_AFFINITY_*: +# From Intel: "The Intel® runtime library has the ability to bind OpenMP +# threads to physical processing units. The interface is controlled using +# the KMP_AFFINITY environment variable. Depending on the system (machine) +# topology, application, and operating system, thread affinity can have a +# dramatic effect on the application speed. +# +# Thread affinity restricts execution of certain threads (virtual execution +# units) to a subset of the physical processing units in a multiprocessor +# computer. Depending upon the topology of the machine, thread affinity can +# have a dramatic effect on the execution speed of a program." +# +# For more information, see the following link: +# https://software.intel.com/content/www/us/en/develop/documentation/cpp- +# compiler-developer-guide-and-reference/top/optimization-and-programming- +# guide/openmp-support/openmp-library-support/thread-affinity-interface- +# linux-and-windows.html +# +# OMP_NUM_THREADS_*: +# The number of OpenMP threads to use for parallel regions. +# +# OMP_STACKSIZE_*: +# Controls the size of the stack for threads created by the OpenMP +# implementation. +# +# Note that settings for the make_grid and make_orog tasks are not +# included below as they do not use parallelized code. +# +#----------------------------------------------------------------------- +# +KMP_AFFINITY_MAKE_OROG: "disabled" +OMP_NUM_THREADS_MAKE_OROG: "6" +OMP_STACKSIZE_MAKE_OROG: "2048m" + +KMP_AFFINITY_MAKE_SFC_CLIMO: "scatter" +OMP_NUM_THREADS_MAKE_SFC_CLIMO: "1" +OMP_STACKSIZE_MAKE_SFC_CLIMO: "1024m" + +KMP_AFFINITY_MAKE_ICS: "scatter" +OMP_NUM_THREADS_MAKE_ICS: "1" +OMP_STACKSIZE_MAKE_ICS: "1024m" + +KMP_AFFINITY_MAKE_LBCS: "scatter" +OMP_NUM_THREADS_MAKE_LBCS: "1" +OMP_STACKSIZE_MAKE_LBCS: "1024m" + +KMP_AFFINITY_RUN_FCST: "scatter" +OMP_NUM_THREADS_RUN_FCST: "2" # atmos_nthreads in model_configure +OMP_STACKSIZE_RUN_FCST: "1024m" + +KMP_AFFINITY_RUN_POST: "scatter" +OMP_NUM_THREADS_RUN_POST: "1" +OMP_STACKSIZE_RUN_POST: "1024m" +# +#----------------------------------------------------------------------- +# diff --git a/ush/constants.py b/ush/constants.py new file mode 100644 index 0000000000..e0ff60a0b0 --- /dev/null +++ b/ush/constants.py @@ -0,0 +1,27 @@ +#!/usr/env/bin python3 + +# +#----------------------------------------------------------------------- +# +# Mathematical and physical constants. +# +#----------------------------------------------------------------------- +# + +# Pi. +pi_geom = 3.14159265358979323846264338327 + +# Degrees per radian. +degs_per_radian = 360.0 / (2.0 * pi_geom) + +# Radius of the Earth in meters. +radius_Earth = 6371200.0 + +# +#----------------------------------------------------------------------- +# +# Other. +# +#----------------------------------------------------------------------- +# +valid_vals_BOOLEAN = [True, False] diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py new file mode 100644 index 0000000000..cd98fcec70 --- /dev/null +++ b/ush/create_diag_table_file.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import os +import unittest +from textwrap import dedent + +from python_utils import import_vars, set_env_var, print_input_args, \ + print_info_msg, print_err_msg_exit, cfg_to_yaml_str + +from fill_jinja_template import fill_jinja_template + +def create_diag_table_file(run_dir): + """ Creates a diagnostic table file for each cycle to be run + + Args: + run_dir: run directory + Returns: + Boolean + """ + + print_input_args(locals()) + + #import all environment variables + import_vars() + + #create a diagnostic table file within the specified run directory + print_info_msg(f''' + Creating a diagnostics table file (\"{DIAG_TABLE_FN}\") in the specified + run directory... + + run_dir = \"{run_dir}\"''', verbose=VERBOSE) + + diag_table_fp = os.path.join(run_dir, DIAG_TABLE_FN) + + print_info_msg(f''' + + Using the template diagnostics table file: + + diag_table_tmpl_fp = {DIAG_TABLE_TMPL_FP} + + to create: + + diag_table_fp = \"{diag_table_fp}\"''', verbose=VERBOSE) + + settings = { + 'starttime': CDATE, + 'cres': CRES + } + settings_str = cfg_to_yaml_str(settings) + + #call fill jinja + try: + fill_jinja_template(["-q", "-u", settings_str, "-t", DIAG_TABLE_TMPL_FP, "-o", diag_table_fp]) + except: + print_err_msg_exit(f''' + !!!!!!!!!!!!!!!!! + + fill_jinja_template.py failed! + + !!!!!!!!!!!!!!!!!''') + return False + return True + +class Testing(unittest.TestCase): + def test_create_diag_table_file(self): + path = os.path.join(os.getenv('USHDIR'), "test_data") + self.assertTrue(create_diag_table_file(run_dir=path)) + def setUp(self): + USHDIR = os.path.dirname(os.path.abspath(__file__)) + DIAG_TABLE_FN="diag_table" + DIAG_TABLE_TMPL_FP = os.path.join(USHDIR,"templates",f"{DIAG_TABLE_FN}.FV3_GFS_v15p2") + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + set_env_var("USHDIR",USHDIR) + set_env_var("DIAG_TABLE_FN",DIAG_TABLE_FN) + set_env_var("DIAG_TABLE_TMPL_FP",DIAG_TABLE_TMPL_FP) + set_env_var("CRES","C48") + set_env_var("CDATE","2021010106") + diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py new file mode 100644 index 0000000000..9ae1241397 --- /dev/null +++ b/ush/create_model_configure_file.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 + +import os +import unittest +from datetime import datetime +from textwrap import dedent + +from python_utils import import_vars, set_env_var, print_input_args, \ + print_info_msg, print_err_msg_exit, lowercase, cfg_to_yaml_str + +from fill_jinja_template import fill_jinja_template + +def create_model_configure_file(cdate,run_dir,sub_hourly_post,dt_subhourly_post_mnts,dt_atmos): + """ Creates a model configuration file in the specified + run directory + + Args: + cdate: cycle date + run_dir: run directory + sub_hourly_post + dt_subhourly_post_mnts + dt_atmos + Returns: + Boolean + """ + + print_input_args(locals()) + + #import all environment variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # Create a model configuration file in the specified run directory. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + Creating a model configuration file (\"{MODEL_CONFIG_FN}\") in the specified + run directory (run_dir): + run_dir = \"{run_dir}\"''', verbose=VERBOSE) + # + # Extract from cdate the starting year, month, day, and hour of the forecast. + # + yyyy=cdate.year + mm=cdate.month + dd=cdate.day + hh=cdate.hour + # + # Set parameters in the model configure file. + # + dot_quilting_dot=f".{lowercase(str(QUILTING))}." + dot_print_esmf_dot=f".{lowercase(str(PRINT_ESMF))}." + dot_cpl_dot=f".{lowercase(str(CPL))}." + dot_write_dopost=f".{lowercase(str(WRITE_DOPOST))}." + # + #----------------------------------------------------------------------- + # + # Create a multiline variable that consists of a yaml-compliant string + # specifying the values that the jinja variables in the template + # model_configure file should be set to. + # + #----------------------------------------------------------------------- + # + settings = { + 'PE_MEMBER01': PE_MEMBER01, + 'print_esmf': dot_print_esmf_dot, + 'start_year': yyyy, + 'start_month': mm, + 'start_day': dd, + 'start_hour': hh, + 'nhours_fcst': FCST_LEN_HRS, + 'dt_atmos': DT_ATMOS, + 'cpl': dot_cpl_dot, + 'atmos_nthreads': OMP_NUM_THREADS_RUN_FCST, + 'restart_interval': RESTART_INTERVAL, + 'write_dopost': dot_write_dopost, + 'quilting': dot_quilting_dot, + 'output_grid': WRTCMP_output_grid + } + # + # If the write-component is to be used, then specify a set of computational + # parameters and a set of grid parameters. The latter depends on the type + # (coordinate system) of the grid that the write-component will be using. + # + if QUILTING: + settings.update({ + 'write_groups': WRTCMP_write_groups, + 'write_tasks_per_group': WRTCMP_write_tasks_per_group, + 'cen_lon': WRTCMP_cen_lon, + 'cen_lat': WRTCMP_cen_lat, + 'lon1': WRTCMP_lon_lwr_left, + 'lat1': WRTCMP_lat_lwr_left + }) + + if WRTCMP_output_grid == "lambert_conformal": + settings.update({ + 'stdlat1': WRTCMP_stdlat1, + 'stdlat2': WRTCMP_stdlat2, + 'nx': WRTCMP_nx, + 'ny': WRTCMP_ny, + 'dx': WRTCMP_dx, + 'dy': WRTCMP_dy, + 'lon2': "", + 'lat2': "", + 'dlon': "", + 'dlat': "", + }) + elif WRTCMP_output_grid == "regional_latlon" or \ + WRTCMP_output_grid == "rotated_latlon": + settings.update({ + 'lon2': WRTCMP_lon_upr_rght, + 'lat2': WRTCMP_lat_upr_rght, + 'dlon': WRTCMP_dlon, + 'dlat': WRTCMP_dlat, + 'stdlat1': "", + 'stdlat2': "", + 'nx': "", + 'ny': "", + 'dx': "", + 'dy': "" + }) + # + # If sub_hourly_post is set to "TRUE", then the forecast model must be + # directed to generate output files on a sub-hourly interval. Do this + # by specifying the output interval in the model configuration file + # (MODEL_CONFIG_FN) in units of number of forecat model time steps (nsout). + # nsout is calculated using the user-specified output time interval + # dt_subhourly_post_mnts (in units of minutes) and the forecast model's + # main time step dt_atmos (in units of seconds). Note that nsout is + # guaranteed to be an integer because the experiment generation scripts + # require that dt_subhourly_post_mnts (after conversion to seconds) be + # evenly divisible by dt_atmos. Also, in this case, the variable output_fh + # [which specifies the output interval in hours; + # see the jinja model_config template file] is set to 0, although this + # doesn't matter because any positive of nsout will override output_fh. + # + # If sub_hourly_post is set to "FALSE", then the workflow is hard-coded + # (in the jinja model_config template file) to direct the forecast model + # to output files every hour. This is done by setting (1) output_fh to 1 + # here, and (2) nsout to -1 here which turns off output by time step interval. + # + # Note that the approach used here of separating how hourly and subhourly + # output is handled should be changed/generalized/simplified such that + # the user should only need to specify the output time interval (there + # should be no need to specify a flag like sub_hourly_post); the workflow + # should then be able to direct the model to output files with that time + # interval and to direct the post-processor to process those files + # regardless of whether that output time interval is larger than, equal + # to, or smaller than one hour. + # + if sub_hourly_post: + nsout=dt_subhourly_post_mnts*60 / dt_atmos + output_fh=0 + else: + output_fh=1 + nsout=-1 + + settings.update({ + 'output_fh': output_fh, + 'nsout': nsout + }) + + settings_str = cfg_to_yaml_str(settings) + + print_info_msg(dedent(f''' + The variable \"settings\" specifying values to be used in the \"{MODEL_CONFIG_FN}\" + file has been set as follows: + #----------------------------------------------------------------------- + settings =\n''') + settings_str,verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # Call a python script to generate the experiment's actual MODEL_CONFIG_FN + # file from the template file. + # + #----------------------------------------------------------------------- + # + model_config_fp=os.path.join(run_dir, MODEL_CONFIG_FN) + + try: + fill_jinja_template(["-q", "-u", settings_str, "-t", MODEL_CONFIG_TMPL_FP, "-o", model_config_fp]) + except: + print_err_msg_exit(f''' + Call to python script fill_jinja_template.py to create a \"{MODEL_CONFIG_FN}\" + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template rocoto XML file: + MODEL_CONFIG_TMPL_FP = \"{MODEL_CONFIG_TMPL_FP}\" + Full path to output rocoto XML file: + model_config_fp = \"{model_config_fp}\" + Namelist settings specified on command line: + settings = + {settings_str}''') + return False + + return True + +class Testing(unittest.TestCase): + def test_create_model_configure_file(self): + path = os.path.join(os.getenv('USHDIR'), "test_data") + self.assertTrue(\ + create_model_configure_file( \ + run_dir=path, + cdate=datetime(2021,1,1), + sub_hourly_post=True, + dt_subhourly_post_mnts=4, + dt_atmos=1) ) + def setUp(self): + USHDIR = os.path.dirname(os.path.abspath(__file__)) + MODEL_CONFIG_FN='model_configure' + MODEL_CONFIG_TMPL_FP = os.path.join(USHDIR, "templates", MODEL_CONFIG_FN) + + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + set_env_var('QUILTING',True) + set_env_var('PRINT_ESMF',True) + set_env_var('CPL',True) + set_env_var('WRITE_DOPOST',True) + set_env_var("USHDIR",USHDIR) + set_env_var('MODEL_CONFIG_FN',MODEL_CONFIG_FN) + set_env_var("MODEL_CONFIG_TMPL_FP",MODEL_CONFIG_TMPL_FP) + set_env_var('PE_MEMBER01',24) + set_env_var('FCST_LEN_HRS',72) + set_env_var('DT_ATMOS',1) + set_env_var('OMP_NUM_THREADS_RUN_FCST',1) + set_env_var('RESTART_INTERVAL',4) + + set_env_var('WRTCMP_write_groups',1) + set_env_var('WRTCMP_write_tasks_per_group',2) + set_env_var('WRTCMP_output_grid',"lambert_conformal") + set_env_var('WRTCMP_cen_lon',-97.5) + set_env_var('WRTCMP_cen_lat',35.0) + set_env_var('WRTCMP_stdlat1',35.0) + set_env_var('WRTCMP_stdlat2',35.0) + set_env_var('WRTCMP_nx',199) + set_env_var('WRTCMP_ny',111) + set_env_var('WRTCMP_lon_lwr_left',-121.23349066) + set_env_var('WRTCMP_lat_lwr_left',23.41731593) + set_env_var('WRTCMP_dx',3000.0) + set_env_var('WRTCMP_dy',3000.0) + + diff --git a/ush/fill_jinja_template.py b/ush/fill_jinja_template.py index 47885051b9..74c3489081 100755 --- a/ush/fill_jinja_template.py +++ b/ush/fill_jinja_template.py @@ -43,6 +43,7 @@ import datetime as dt import os +import sys import argparse import jinja2 as j2 @@ -155,7 +156,7 @@ def path_ok(arg): raise argparse.ArgumentTypeError(msg) -def parse_args(): +def parse_args(argv): ''' Function maintains the arguments accepted by this script. Please see @@ -195,7 +196,7 @@ def parse_args(): required=True, type=path_ok, ) - return parser.parse_args() + return parser.parse_args(argv) def update_dict(dest, newdict, quiet=False): @@ -231,13 +232,18 @@ def update_dict(dest, newdict, quiet=False): print('*' * 50) -def main(cla): +def fill_jinja_template(argv): ''' Loads a Jinja template, determines its necessary undefined variables, retrives them from user supplied settings, and renders the final result. ''' + # parse args + cla = parse_args(argv) + if cla.config: + cla.config = config_exists(cla.config) + # Create a Jinja Environment to load the template. env = j2.Environment(loader=j2.FileSystemLoader(cla.template)) template_source = env.loader.get_source(env, '') @@ -274,7 +280,5 @@ def main(cla): if __name__ == '__main__': - cla = parse_args() - if cla.config: - cla.config = config_exists(cla.config) - main(cla) + + fill_jinja_template(sys.argv[1:]) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py new file mode 100755 index 0000000000..16a2a63425 --- /dev/null +++ b/ush/generate_FV3LAM_wflow.py @@ -0,0 +1,1126 @@ +#!/usr/bin/env python3 + +import os +import sys +import platform +import subprocess +from multiprocessing import Process +from textwrap import dedent +from datetime import datetime,timedelta + +from python_utils import print_info_msg, print_err_msg_exit, import_vars, cp_vrfy, cd_vrfy,\ + rm_vrfy, ln_vrfy, mkdir_vrfy, mv_vrfy, run_command, date_to_str, \ + define_macos_utilities, create_symlink_to_file, check_for_preexist_dir_file, \ + cfg_to_yaml_str, find_pattern_in_str + +from setup import setup +from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames +from get_crontab_contents import get_crontab_contents +from fill_jinja_template import fill_jinja_template +from set_namelist import set_namelist + +def python_error_handler(): + """ Error handler for missing packages """ + + print_err_msg_exit(''' + Errors found: check your python environment + + Instructions for setting up python environments can be found on the web: + https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started + ''', stack_trace=False) + +# Check for non-standard python packages +try: + import jinja2 + import yaml + import f90nml +except ImportError as error: + print_info_msg(error.__class__.__name__ + ": " + str(error)) + python_error_handler() + +def generate_FV3LAM_wflow(): + """ Function to setup a forecast experiment and create a workflow + (according to the parameters specified in the config file + + Args: + None + Returns: + None + """ + + print(dedent(''' + ======================================================================== + ======================================================================== + + Starting experiment generation... + + ======================================================================== + ========================================================================''')) + + #set ushdir + ushdir = os.path.dirname(os.path.abspath(__file__)) + + #check python version + major,minor,patch = platform.python_version_tuple() + if int(major) < 3 or int(minor) < 6: + print_info_msg(f''' + + Error: python version must be 3.6 or higher + python version: {major}.{minor}''') + + #define macros + define_macos_utilities() + + # + #----------------------------------------------------------------------- + # + # Source the file that defines and then calls the setup function. The + # setup function in turn first sources the default configuration file + # (which contains default values for the experiment/workflow parameters) + # and then sources the user-specified configuration file (which contains + # user-specified values for a subset of the experiment/workflow parame- + # ters that override their default values). + # + #----------------------------------------------------------------------- + # + setup() + + #import all environment variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # Set the full path to the experiment's rocoto workflow xml file. This + # file will be placed at the top level of the experiment directory and + # then used by rocoto to run the workflow. + # + #----------------------------------------------------------------------- + # + WFLOW_XML_FP = os.path.join(EXPTDIR, WFLOW_XML_FN) + + # + #----------------------------------------------------------------------- + # + # Create a multiline variable that consists of a yaml-compliant string + # specifying the values that the jinja variables in the template rocoto + # XML should be set to. These values are set either in the user-specified + # workflow configuration file (EXPT_CONFIG_FN) or in the setup.sh script + # sourced above. Then call the python script that generates the XML. + # + #----------------------------------------------------------------------- + # + if WORKFLOW_MANAGER == "rocoto": + + template_xml_fp = os.path.join(TEMPLATE_DIR, WFLOW_XML_FN) + + print_info_msg(f''' + Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML + file (template_xml_fp): + template_xml_fp = \"{template_xml_fp}\" + WFLOW_XML_FP = \"{WFLOW_XML_FP}\"''') + + ensmem_indx_name = "" + uscore_ensmem_name = "" + slash_ensmem_subdir = "" + if DO_ENSEMBLE: + ensmem_indx_name = "mem" + uscore_ensmem_name = f"_mem#{ensmem_indx_name}#" + slash_ensmem_subdir = f"/mem#{ensmem_indx_name}#" + + #get time string + d = DATE_FIRST_CYCL + timedelta(seconds=DT_ATMOS) + time_str = d.strftime("%M:%S") + + cycl_hrs_str = [ f"{c:02d}" for c in CYCL_HRS ] + cdate_first_cycl = DATE_FIRST_CYCL + timedelta(hours=CYCL_HRS[0]) + + # Dictionary of settings + settings = { + # + # Parameters needed by the job scheduler. + # + 'account': ACCOUNT, + 'sched': SCHED, + 'partition_default': PARTITION_DEFAULT, + 'queue_default': QUEUE_DEFAULT, + 'partition_hpss': PARTITION_HPSS, + 'queue_hpss': QUEUE_HPSS, + 'partition_fcst': PARTITION_FCST, + 'queue_fcst': QUEUE_FCST, + 'machine': MACHINE, + 'slurm_native_cmd': SLURM_NATIVE_CMD, + # + # Workflow task names. + # + 'make_grid_tn': MAKE_GRID_TN, + 'make_orog_tn': MAKE_OROG_TN, + 'make_sfc_climo_tn': MAKE_SFC_CLIMO_TN, + 'get_extrn_ics_tn': GET_EXTRN_ICS_TN, + 'get_extrn_lbcs_tn': GET_EXTRN_LBCS_TN, + 'make_ics_tn': MAKE_ICS_TN, + 'make_lbcs_tn': MAKE_LBCS_TN, + 'run_fcst_tn': RUN_FCST_TN, + 'run_post_tn': RUN_POST_TN, + 'get_obs_ccpa_tn': GET_OBS_CCPA_TN, + 'get_obs_ndas_tn': GET_OBS_NDAS_TN, + 'get_obs_mrms_tn': GET_OBS_MRMS_TN, + 'vx_tn': VX_TN, + 'vx_gridstat_tn': VX_GRIDSTAT_TN, + 'vx_gridstat_refc_tn': VX_GRIDSTAT_REFC_TN, + 'vx_gridstat_retop_tn': VX_GRIDSTAT_RETOP_TN, + 'vx_gridstat_03h_tn': VX_GRIDSTAT_03h_TN, + 'vx_gridstat_06h_tn': VX_GRIDSTAT_06h_TN, + 'vx_gridstat_24h_tn': VX_GRIDSTAT_24h_TN, + 'vx_pointstat_tn': VX_POINTSTAT_TN, + 'vx_ensgrid_tn': VX_ENSGRID_TN, + 'vx_ensgrid_refc_tn': VX_ENSGRID_REFC_TN, + 'vx_ensgrid_retop_tn': VX_ENSGRID_RETOP_TN, + 'vx_ensgrid_03h_tn': VX_ENSGRID_03h_TN, + 'vx_ensgrid_06h_tn': VX_ENSGRID_06h_TN, + 'vx_ensgrid_24h_tn': VX_ENSGRID_24h_TN, + 'vx_ensgrid_mean_tn': VX_ENSGRID_MEAN_TN, + 'vx_ensgrid_prob_tn': VX_ENSGRID_PROB_TN, + 'vx_ensgrid_mean_03h_tn': VX_ENSGRID_MEAN_03h_TN, + 'vx_ensgrid_prob_03h_tn': VX_ENSGRID_PROB_03h_TN, + 'vx_ensgrid_mean_06h_tn': VX_ENSGRID_MEAN_06h_TN, + 'vx_ensgrid_prob_06h_tn': VX_ENSGRID_PROB_06h_TN, + 'vx_ensgrid_mean_24h_tn': VX_ENSGRID_MEAN_24h_TN, + 'vx_ensgrid_prob_24h_tn': VX_ENSGRID_PROB_24h_TN, + 'vx_ensgrid_prob_refc_tn': VX_ENSGRID_PROB_REFC_TN, + 'vx_ensgrid_prob_retop_tn': VX_ENSGRID_PROB_RETOP_TN, + 'vx_enspoint_tn': VX_ENSPOINT_TN, + 'vx_enspoint_mean_tn': VX_ENSPOINT_MEAN_TN, + 'vx_enspoint_prob_tn': VX_ENSPOINT_PROB_TN, + # + # Entity used to load the module file for each GET_OBS_* task. + # + 'get_obs': GET_OBS, + # + # Number of nodes to use for each task. + # + 'nnodes_make_grid': NNODES_MAKE_GRID, + 'nnodes_make_orog': NNODES_MAKE_OROG, + 'nnodes_make_sfc_climo': NNODES_MAKE_SFC_CLIMO, + 'nnodes_get_extrn_ics': NNODES_GET_EXTRN_ICS, + 'nnodes_get_extrn_lbcs': NNODES_GET_EXTRN_LBCS, + 'nnodes_make_ics': NNODES_MAKE_ICS, + 'nnodes_make_lbcs': NNODES_MAKE_LBCS, + 'nnodes_run_fcst': NNODES_RUN_FCST, + 'nnodes_run_post': NNODES_RUN_POST, + 'nnodes_get_obs_ccpa': NNODES_GET_OBS_CCPA, + 'nnodes_get_obs_mrms': NNODES_GET_OBS_MRMS, + 'nnodes_get_obs_ndas': NNODES_GET_OBS_NDAS, + 'nnodes_vx_gridstat': NNODES_VX_GRIDSTAT, + 'nnodes_vx_pointstat': NNODES_VX_POINTSTAT, + 'nnodes_vx_ensgrid': NNODES_VX_ENSGRID, + 'nnodes_vx_ensgrid_mean': NNODES_VX_ENSGRID_MEAN, + 'nnodes_vx_ensgrid_prob': NNODES_VX_ENSGRID_PROB, + 'nnodes_vx_enspoint': NNODES_VX_ENSPOINT, + 'nnodes_vx_enspoint_mean': NNODES_VX_ENSPOINT_MEAN, + 'nnodes_vx_enspoint_prob': NNODES_VX_ENSPOINT_PROB, + # + # Number of cores used for a task + # + 'ncores_run_fcst': PE_MEMBER01, + 'native_run_fcst': f"--cpus-per-task {OMP_NUM_THREADS_RUN_FCST} --exclusive", + # + # Number of logical processes per node for each task. If running without + # threading, this is equal to the number of MPI processes per node. + # + 'ppn_make_grid': PPN_MAKE_GRID, + 'ppn_make_orog': PPN_MAKE_OROG, + 'ppn_make_sfc_climo': PPN_MAKE_SFC_CLIMO, + 'ppn_get_extrn_ics': PPN_GET_EXTRN_ICS, + 'ppn_get_extrn_lbcs': PPN_GET_EXTRN_LBCS, + 'ppn_make_ics': PPN_MAKE_ICS, + 'ppn_make_lbcs': PPN_MAKE_LBCS, + 'ppn_run_fcst': PPN_RUN_FCST, + 'ppn_run_post': PPN_RUN_POST, + 'ppn_get_obs_ccpa': PPN_GET_OBS_CCPA, + 'ppn_get_obs_mrms': PPN_GET_OBS_MRMS, + 'ppn_get_obs_ndas': PPN_GET_OBS_NDAS, + 'ppn_vx_gridstat': PPN_VX_GRIDSTAT, + 'ppn_vx_pointstat': PPN_VX_POINTSTAT, + 'ppn_vx_ensgrid': PPN_VX_ENSGRID, + 'ppn_vx_ensgrid_mean': PPN_VX_ENSGRID_MEAN, + 'ppn_vx_ensgrid_prob': PPN_VX_ENSGRID_PROB, + 'ppn_vx_enspoint': PPN_VX_ENSPOINT, + 'ppn_vx_enspoint_mean': PPN_VX_ENSPOINT_MEAN, + 'ppn_vx_enspoint_prob': PPN_VX_ENSPOINT_PROB, + # + # Maximum wallclock time for each task. + # + 'wtime_make_grid': WTIME_MAKE_GRID, + 'wtime_make_orog': WTIME_MAKE_OROG, + 'wtime_make_sfc_climo': WTIME_MAKE_SFC_CLIMO, + 'wtime_get_extrn_ics': WTIME_GET_EXTRN_ICS, + 'wtime_get_extrn_lbcs': WTIME_GET_EXTRN_LBCS, + 'wtime_make_ics': WTIME_MAKE_ICS, + 'wtime_make_lbcs': WTIME_MAKE_LBCS, + 'wtime_run_fcst': WTIME_RUN_FCST, + 'wtime_run_post': WTIME_RUN_POST, + 'wtime_get_obs_ccpa': WTIME_GET_OBS_CCPA, + 'wtime_get_obs_mrms': WTIME_GET_OBS_MRMS, + 'wtime_get_obs_ndas': WTIME_GET_OBS_NDAS, + 'wtime_vx_gridstat': WTIME_VX_GRIDSTAT, + 'wtime_vx_pointstat': WTIME_VX_POINTSTAT, + 'wtime_vx_ensgrid': WTIME_VX_ENSGRID, + 'wtime_vx_ensgrid_mean': WTIME_VX_ENSGRID_MEAN, + 'wtime_vx_ensgrid_prob': WTIME_VX_ENSGRID_PROB, + 'wtime_vx_enspoint': WTIME_VX_ENSPOINT, + 'wtime_vx_enspoint_mean': WTIME_VX_ENSPOINT_MEAN, + 'wtime_vx_enspoint_prob': WTIME_VX_ENSPOINT_PROB, + # + # Maximum number of tries for each task. + # + 'maxtries_make_grid': MAXTRIES_MAKE_GRID, + 'maxtries_make_orog': MAXTRIES_MAKE_OROG, + 'maxtries_make_sfc_climo': MAXTRIES_MAKE_SFC_CLIMO, + 'maxtries_get_extrn_ics': MAXTRIES_GET_EXTRN_ICS, + 'maxtries_get_extrn_lbcs': MAXTRIES_GET_EXTRN_LBCS, + 'maxtries_make_ics': MAXTRIES_MAKE_ICS, + 'maxtries_make_lbcs': MAXTRIES_MAKE_LBCS, + 'maxtries_run_fcst': MAXTRIES_RUN_FCST, + 'maxtries_run_post': MAXTRIES_RUN_POST, + 'maxtries_get_obs_ccpa': MAXTRIES_GET_OBS_CCPA, + 'maxtries_get_obs_mrms': MAXTRIES_GET_OBS_MRMS, + 'maxtries_get_obs_ndas': MAXTRIES_GET_OBS_NDAS, + 'maxtries_vx_gridstat': MAXTRIES_VX_GRIDSTAT, + 'maxtries_vx_gridstat_refc': MAXTRIES_VX_GRIDSTAT_REFC, + 'maxtries_vx_gridstat_retop': MAXTRIES_VX_GRIDSTAT_RETOP, + 'maxtries_vx_gridstat_03h': MAXTRIES_VX_GRIDSTAT_03h, + 'maxtries_vx_gridstat_06h': MAXTRIES_VX_GRIDSTAT_06h, + 'maxtries_vx_gridstat_24h': MAXTRIES_VX_GRIDSTAT_24h, + 'maxtries_vx_pointstat': MAXTRIES_VX_POINTSTAT, + 'maxtries_vx_ensgrid': MAXTRIES_VX_ENSGRID, + 'maxtries_vx_ensgrid_refc': MAXTRIES_VX_ENSGRID_REFC, + 'maxtries_vx_ensgrid_retop': MAXTRIES_VX_ENSGRID_RETOP, + 'maxtries_vx_ensgrid_03h': MAXTRIES_VX_ENSGRID_03h, + 'maxtries_vx_ensgrid_06h': MAXTRIES_VX_ENSGRID_06h, + 'maxtries_vx_ensgrid_24h': MAXTRIES_VX_ENSGRID_24h, + 'maxtries_vx_ensgrid_mean': MAXTRIES_VX_ENSGRID_MEAN, + 'maxtries_vx_ensgrid_prob': MAXTRIES_VX_ENSGRID_PROB, + 'maxtries_vx_ensgrid_mean_03h': MAXTRIES_VX_ENSGRID_MEAN_03h, + 'maxtries_vx_ensgrid_prob_03h': MAXTRIES_VX_ENSGRID_PROB_03h, + 'maxtries_vx_ensgrid_mean_06h': MAXTRIES_VX_ENSGRID_MEAN_06h, + 'maxtries_vx_ensgrid_prob_06h': MAXTRIES_VX_ENSGRID_PROB_06h, + 'maxtries_vx_ensgrid_mean_24h': MAXTRIES_VX_ENSGRID_MEAN_24h, + 'maxtries_vx_ensgrid_prob_24h': MAXTRIES_VX_ENSGRID_PROB_24h, + 'maxtries_vx_ensgrid_prob_refc': MAXTRIES_VX_ENSGRID_PROB_REFC, + 'maxtries_vx_ensgrid_prob_retop': MAXTRIES_VX_ENSGRID_PROB_RETOP, + 'maxtries_vx_enspoint': MAXTRIES_VX_ENSPOINT, + 'maxtries_vx_enspoint_mean': MAXTRIES_VX_ENSPOINT_MEAN, + 'maxtries_vx_enspoint_prob': MAXTRIES_VX_ENSPOINT_PROB, + # + # Flags that specify whether to run the preprocessing or + # verification-related tasks. + # + 'run_task_make_grid': RUN_TASK_MAKE_GRID, + 'run_task_make_orog': RUN_TASK_MAKE_OROG, + 'run_task_make_sfc_climo': RUN_TASK_MAKE_SFC_CLIMO, + 'run_task_get_extrn_ics': RUN_TASK_GET_EXTRN_ICS, + 'run_task_get_extrn_lbcs': RUN_TASK_GET_EXTRN_LBCS, + 'run_task_make_ics': RUN_TASK_MAKE_ICS, + 'run_task_make_lbcs': RUN_TASK_MAKE_LBCS, + 'run_task_run_fcst': RUN_TASK_RUN_FCST, + 'run_task_run_post': RUN_TASK_RUN_POST, + 'run_task_get_obs_ccpa': RUN_TASK_GET_OBS_CCPA, + 'run_task_get_obs_mrms': RUN_TASK_GET_OBS_MRMS, + 'run_task_get_obs_ndas': RUN_TASK_GET_OBS_NDAS, + 'run_task_vx_gridstat': RUN_TASK_VX_GRIDSTAT, + 'run_task_vx_pointstat': RUN_TASK_VX_POINTSTAT, + 'run_task_vx_ensgrid': RUN_TASK_VX_ENSGRID, + 'run_task_vx_enspoint': RUN_TASK_VX_ENSPOINT, + # + # Number of physical cores per node for the current machine. + # + 'ncores_per_node': NCORES_PER_NODE, + # + # Directories and files. + # + 'jobsdir': JOBSDIR, + 'logdir': LOGDIR, + 'scriptsdir': SCRIPTSDIR, + 'cycle_basedir': CYCLE_BASEDIR, + 'global_var_defns_fp': GLOBAL_VAR_DEFNS_FP, + 'load_modules_run_task_fp': LOAD_MODULES_RUN_TASK_FP, + # + # External model information for generating ICs and LBCs. + # + 'extrn_mdl_name_ics': EXTRN_MDL_NAME_ICS, + 'extrn_mdl_name_lbcs': EXTRN_MDL_NAME_LBCS, + # + # Parameters that determine the set of cycles to run. + # + 'date_first_cycl': date_to_str(DATE_FIRST_CYCL,True), + 'date_last_cycl': date_to_str(DATE_LAST_CYCL,True), + 'cdate_first_cycl': cdate_first_cycl, + 'cycl_hrs': cycl_hrs_str, + 'cycl_freq': f"{INCR_CYCL_FREQ:02d}:00:00", + # + # Forecast length (same for all cycles). + # + 'fcst_len_hrs': FCST_LEN_HRS, + # + # Inline post + # + 'write_dopost': WRITE_DOPOST, + # + # METPlus-specific information + # + 'model': MODEL, + 'met_install_dir': MET_INSTALL_DIR, + 'met_bin_exec': MET_BIN_EXEC, + 'metplus_path': METPLUS_PATH, + 'vx_config_dir': VX_CONFIG_DIR, + 'metplus_conf': METPLUS_CONF, + 'met_config': MET_CONFIG, + 'ccpa_obs_dir': CCPA_OBS_DIR, + 'mrms_obs_dir': MRMS_OBS_DIR, + 'ndas_obs_dir': NDAS_OBS_DIR, + # + # Ensemble-related parameters. + # + 'do_ensemble': DO_ENSEMBLE, + 'num_ens_members': NUM_ENS_MEMBERS, + 'ndigits_ensmem_names': f"{NDIGITS_ENSMEM_NAMES}", + 'ensmem_indx_name': ensmem_indx_name, + 'uscore_ensmem_name': uscore_ensmem_name, + 'slash_ensmem_subdir': slash_ensmem_subdir, + # + # Parameters associated with subhourly post-processed output + # + 'sub_hourly_post': SUB_HOURLY_POST, + 'delta_min': DT_SUBHOURLY_POST_MNTS, + 'first_fv3_file_tstr': f"000:{time_str}" + } + # End of "settings" variable. + settings_str = cfg_to_yaml_str(settings) + + print_info_msg(dedent(f''' + The variable \"settings\" specifying values of the rococo XML variables + has been set as follows: + #----------------------------------------------------------------------- + settings =\n''') + settings_str, verbose=VERBOSE) + + # + # Call the python script to generate the experiment's actual XML file + # from the jinja template file. + # + try: + fill_jinja_template(["-q", "-u", settings_str, "-t", template_xml_fp, "-o", WFLOW_XML_FP]) + except: + print_err_msg_exit(f''' + Call to python script fill_jinja_template.py to create a rocoto workflow + XML file from a template file failed. Parameters passed to this script + are: + Full path to template rocoto XML file: + template_xml_fp = \"{template_xml_fp}\" + Full path to output rocoto XML file: + WFLOW_XML_FP = \"{WFLOW_XML_FP}\" + Namelist settings specified on command line: + settings = + {settings_str}''') + # + #----------------------------------------------------------------------- + # + # Create a symlink in the experiment directory that points to the workflow + # (re)launch script. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + Creating symlink in the experiment directory (EXPTDIR) that points to the + workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): + EXPTDIR = \"{EXPTDIR}\" + WFLOW_LAUNCH_SCRIPT_FP = \"{WFLOW_LAUNCH_SCRIPT_FP}\"''',verbose=VERBOSE) + + create_symlink_to_file(WFLOW_LAUNCH_SCRIPT_FP, + os.path.join(EXPTDIR,WFLOW_LAUNCH_SCRIPT_FN), + False) + # + #----------------------------------------------------------------------- + # + # If USE_CRON_TO_RELAUNCH is set to TRUE, add a line to the user's cron + # table to call the (re)launch script every CRON_RELAUNCH_INTVL_MNTS mi- + # nutes. + # + #----------------------------------------------------------------------- + # + if USE_CRON_TO_RELAUNCH: + # + # Make a backup copy of the user's crontab file and save it in a file. + # + time_stamp = datetime.now().strftime("%F_%T") + crontab_backup_fp=os.path.join(EXPTDIR,f"crontab.bak.{time_stamp}") + print_info_msg(f''' + Copying contents of user cron table to backup file: + crontab_backup_fp = \"{crontab_backup_fp}\"''',verbose=VERBOSE) + + global called_from_cron + try: called_from_cron + except: called_from_cron = False + + crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=called_from_cron) + # To create the backup crontab file and add a new job to the user's + # existing cron table, use the "printf" command, not "echo", to print + # out variables. This is because "echo" will add a newline at the end + # of its output even if its input argument is a null string, resulting + # in extranous blank lines in the backup crontab file and/or the cron + # table itself. Using "printf" prevents the appearance of these blank + # lines. + run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''') + # + # Below, we use "grep" to determine whether the crontab line that the + # variable CRONTAB_LINE contains is already present in the cron table. + # For that purpose, we need to escape the asterisks in the string in + # CRONTAB_LINE with backslashes. Do this next. + # + (_,crontab_line_esc_astr,_) = run_command(f'''printf "%s" '{CRONTAB_LINE}' | \ + {SED} -r -e "s%[*]%\\\\*%g"''') + # In the grep command below, the "^" at the beginning of the string + # passed to grep is a start-of-line anchor, and the "$" at the end is + # an end-of-line anchor. Thus, in order for grep to find a match on + # any given line of the cron table's contents, that line must contain + # exactly the string in the variable crontab_line_esc_astr without any + # leading or trailing characters. This is to eliminate situations in + # which a line in the cron table contains the string in crontab_line_esc_astr + # but is precedeeded, for example, by the comment character "#" (in which + # case cron ignores that line) and/or is followed by further commands + # that are not part of the string in crontab_line_esc_astr (in which + # case it does something more than the command portion of the string in + # crontab_line_esc_astr does). + # + if MACHINE == "WCOSS_DELL_P3": + (exit_status,grep_output,_)=run_command(f'''grep '^{crontab_line_esc_astr}$' "/u/{USER}/cron/mycrontab"''') + else: + (exit_status,grep_output,_)=run_command(f'''printf "%s" '{crontab_contents}' | grep "^{crontab_line_esc_astr}$"''') + + if exit_status == 0: + + print_info_msg(f''' + The following line already exists in the cron table and thus will not be + added: + CRONTAB_LINE = \"{CRONTAB_LINE}\"''') + + else: + + print_info_msg(f''' + Adding the following line to the user's cron table in order to automatically + resubmit SRW workflow: + CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE) + + if MACHINE == "WCOSS_DELL_P3": + run_command(f'''printf "%s" '{CRONTAB_LINE}' >> f"/u/{USER}/cron/mycrontab"''') + else: + # Add a newline to the end of crontab_contents only if it is not empty. + # This is needed so that when CRONTAB_LINE is printed out, it appears on + # a separate line. + if crontab_contents: + crontab_contents += "\n" + run_command(f'''( printf "%s" '{crontab_contents}'; printf "%s\n" '{CRONTAB_LINE}' ) | {crontab_cmd}''') + # + #----------------------------------------------------------------------- + # + # Create the FIXam directory under the experiment directory. In NCO mode, + # this will be a symlink to the directory specified in FIXgsm, while in + # community mode, it will be an actual directory with files copied into + # it from FIXgsm. + # + #----------------------------------------------------------------------- + # + # First, consider NCO mode. + # + if RUN_ENVIR == "nco": + + ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') + # + # Resolve the target directory that the FIXam symlink points to and check + # that it exists. + # + try: + path_resolved = os.path.realpath(FIXam) + except: + path_resolved = FIXam + if not os.path.exists(path_resolved): + print_err_msg_exit(f''' + In order to be able to generate a forecast experiment in NCO mode (i.e. + when RUN_ENVIR set to \"nco\"), the path specified by FIXam after resolving + all symlinks (path_resolved) must be an existing directory (but in this + case isn't): + RUN_ENVIR = \"{RUN_ENVIR}\" + FIXam = \"{FIXam}\" + path_resolved = \"{path_resolved}\" + Please ensure that path_resolved is an existing directory and then rerun + the experiment generation script.''') + # + # Now consider community mode. + # + else: + + print_info_msg(f''' + Copying fixed files from system directory (FIXgsm) to a subdirectory + (FIXam) in the experiment directory: + FIXgsm = \"{FIXgsm}\" + FIXam = \"{FIXam}\"''',verbose=VERBOSE) + + check_for_preexist_dir_file(FIXam,"delete") + mkdir_vrfy("-p",FIXam) + mkdir_vrfy("-p",os.path.join(FIXam,"fix_co2_proj")) + + num_files=len(FIXgsm_FILES_TO_COPY_TO_FIXam) + for i in range(num_files): + fn=f"{FIXgsm_FILES_TO_COPY_TO_FIXam[i]}" + cp_vrfy(os.path.join(FIXgsm,fn), os.path.join(FIXam,fn)) + # + #----------------------------------------------------------------------- + # + # Copy MERRA2 aerosol climatology data. + # + #----------------------------------------------------------------------- + # + if USE_MERRA_CLIMO: + print_info_msg(f''' + Copying MERRA2 aerosol climatology data files from system directory + (FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: + FIXaer = \"{FIXaer}\" + FIXlut = \"{FIXlut}\" + FIXclim = \"{FIXclim}\"''',verbose=VERBOSE) + + check_for_preexist_dir_file(FIXclim,"delete") + mkdir_vrfy("-p", FIXclim) + + cp_vrfy(os.path.join(FIXaer,"merra2.aerclim*.nc"), FIXclim) + cp_vrfy(os.path.join(FIXlut,"optics*.dat"), FIXclim) + # + #----------------------------------------------------------------------- + # + # Copy templates of various input files to the experiment directory. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + Copying templates of various input files to the experiment directory...''', + verbose=VERBOSE) + + print_info_msg(f''' + Copying the template data table file to the experiment directory...''', + verbose=VERBOSE) + cp_vrfy(DATA_TABLE_TMPL_FP,DATA_TABLE_FP) + + print_info_msg(f''' + Copying the template field table file to the experiment directory...''', + verbose=VERBOSE) + cp_vrfy(FIELD_TABLE_TMPL_FP,FIELD_TABLE_FP) + + print_info_msg(f''' + Copying the template NEMS configuration file to the experiment directory...''', + verbose=VERBOSE) + cp_vrfy(NEMS_CONFIG_TMPL_FP,NEMS_CONFIG_FP) + # + # Copy the CCPP physics suite definition file from its location in the + # clone of the FV3 code repository to the experiment directory (EXPT- + # DIR). + # + print_info_msg(f''' + Copying the CCPP physics suite definition XML file from its location in + the forecast model directory sturcture to the experiment directory...''', + verbose=VERBOSE) + cp_vrfy(CCPP_PHYS_SUITE_IN_CCPP_FP,CCPP_PHYS_SUITE_FP) + # + # Copy the field dictionary file from its location in the + # clone of the FV3 code repository to the experiment directory (EXPT- + # DIR). + # + print_info_msg(f''' + Copying the field dictionary file from its location in the forecast + model directory sturcture to the experiment directory...''', + verbose=VERBOSE) + cp_vrfy(FIELD_DICT_IN_UWM_FP,FIELD_DICT_FP) + # + #----------------------------------------------------------------------- + # + # Set parameters in the FV3-LAM namelist file. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + Setting parameters in weather model's namelist file (FV3_NML_FP): + FV3_NML_FP = \"{FV3_NML_FP}\"''') + # + # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. + # These need to be set in the FV3-LAM Fortran namelist file. They represent + # the number of cell vertices in the x and y directions on the regional + # grid. + # + npx=NX+1 + npy=NY+1 + # + # For the physics suites that use RUC LSM, set the parameter kice to 9, + # Otherwise, leave it unspecified (which means it gets set to the default + # value in the forecast model). + # + # NOTE: + # May want to remove kice from FV3.input.yml (and maybe input.nml.FV3). + # + kice=None + if SDF_USES_RUC_LSM: + kice=9 + # + # Set lsoil, which is the number of input soil levels provided in the + # chgres_cube output NetCDF file. This is the same as the parameter + # nsoill_out in the namelist file for chgres_cube. [On the other hand, + # the parameter lsoil_lsm (not set here but set in input.nml.FV3 and/or + # FV3.input.yml) is the number of soil levels that the LSM scheme in the + # forecast model will run with.] Here, we use the same approach to set + # lsoil as the one used to set nsoill_out in exregional_make_ics.sh. + # See that script for details. + # + # NOTE: + # May want to remove lsoil from FV3.input.yml (and maybe input.nml.FV3). + # Also, may want to set lsm here as well depending on SDF_USES_RUC_LSM. + # + lsoil=4 + if ( EXTRN_MDL_NAME_ICS == "HRRR" or \ + EXTRN_MDL_NAME_ICS == "RAP" ) and \ + ( SDF_USES_RUC_LSM ): + lsoil=9 + # + # Create a multiline variable that consists of a yaml-compliant string + # specifying the values that the namelist variables that are physics- + # suite-independent need to be set to. Below, this variable will be + # passed to a python script that will in turn set the values of these + # variables in the namelist file. + # + # IMPORTANT: + # If we want a namelist variable to be removed from the namelist file, + # in the "settings" variable below, we need to set its value to the + # string "null". This is equivalent to setting its value to + # !!python/none + # in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the + # suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. + # + # It turns out that setting the variable to an empty string also works + # to remove it from the namelist! Which is better to use?? + # + settings = {} + settings['atmos_model_nml'] = { + 'blocksize': BLOCKSIZE, + 'ccpp_suite': CCPP_PHYS_SUITE + } + settings['fv_core_nml'] = { + 'target_lon': LON_CTR, + 'target_lat': LAT_CTR, + 'nrows_blend': HALO_BLEND, + # + # Question: + # For a ESGgrid type grid, what should stretch_fac be set to? This depends + # on how the FV3 code uses the stretch_fac parameter in the namelist file. + # Recall that for a ESGgrid, it gets set in the function set_gridparams_ESGgrid(.sh) + # to something like 0.9999, but is it ok to set it to that here in the + # FV3 namelist file? + # + 'stretch_fac': STRETCH_FAC, + 'npx': npx, + 'npy': npy, + 'layout': [LAYOUT_X, LAYOUT_Y], + 'bc_update_interval': LBC_SPEC_INTVL_HRS + } + settings['gfs_physics_nml'] = { + 'kice': kice or None, + 'lsoil': lsoil or None, + 'do_shum': DO_SHUM, + 'do_sppt': DO_SPPT, + 'do_skeb': DO_SKEB, + 'do_spp': DO_SPP, + 'n_var_spp': N_VAR_SPP, + 'n_var_lndp': N_VAR_LNDP, + 'lndp_type': LNDP_TYPE, + 'lndp_each_step': LSM_SPP_EACH_STEP, + 'fhcyc': FHCYC_LSM_SPP_OR_NOT + } + # + # Add to "settings" the values of those namelist variables that specify + # the paths to fixed files in the FIXam directory. As above, these namelist + # variables are physcs-suite-independent. + # + # Note that the array FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING contains + # the mapping between the namelist variables and the names of the files + # in the FIXam directory. Here, we loop through this array and process + # each element to construct each line of "settings". + # + dummy_run_dir=os.path.join(EXPTDIR,"any_cyc") + if DO_ENSEMBLE: + dummy_run_dir=os.path.join(dummy_run_dir,"any_ensmem") + + regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" + num_nml_vars=len(FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING) + namsfc_dict = {} + for i in range(num_nml_vars): + + mapping=f"{FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[i]}" + tup = find_pattern_in_str(regex_search, mapping) + nml_var_name = tup[0] + FIXam_fn = tup[1] + + fp="\"\"" + if FIXam_fn: + fp=os.path.join(FIXam,FIXam_fn) + # + # If not in NCO mode, for portability and brevity, change fp so that it + # is a relative path (relative to any cycle directory immediately under + # the experiment directory). + # + if RUN_ENVIR != "nco": + fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) + # + # Add a line to the variable "settings" that specifies (in a yaml-compliant + # format) the name of the current namelist variable and the value it should + # be set to. + # + namsfc_dict[nml_var_name] = fp + # + # Add namsfc_dict to settings + # + settings['namsfc'] = namsfc_dict + # + # Use netCDF4 when running the North American 3-km domain due to file size. + # + if PREDEF_GRID_NAME == "RRFS_NA_3km": + settings['fms2_io_nml'] = { + 'netcdf_default_format': 'netcdf4' + } + # + # Add the relevant tendency-based stochastic physics namelist variables to + # "settings" when running with SPPT, SHUM, or SKEB turned on. If running + # with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only + # include an empty "nam_stochy" stanza. + # + nam_stochy_dict = {} + if DO_SPPT: + nam_stochy_dict.update({ + 'iseed_sppt': ISEED_SPPT, + 'new_lscale': NEW_LSCALE, + 'sppt': SPPT_MAG, + 'sppt_logit': SPPT_LOGIT, + 'sppt_lscale': SPPT_LSCALE, + 'sppt_sfclimit': SPPT_SFCLIMIT, + 'sppt_tau': SPPT_TSCALE, + 'spptint': SPPT_INT, + 'use_zmtnblck': USE_ZMTNBLCK + }) + + if DO_SHUM: + nam_stochy_dict.update({ + 'iseed_shum': ISEED_SHUM, + 'new_lscale': NEW_LSCALE, + 'shum': SHUM_MAG, + 'shum_lscale': SHUM_LSCALE, + 'shum_tau': SHUM_TSCALE, + 'shumint': SHUM_INT + }) + + if DO_SKEB: + nam_stochy_dict.update({ + 'iseed_skeb': ISEED_SKEB, + 'new_lscale': NEW_LSCALE, + 'skeb': SKEB_MAG, + 'skeb_lscale': SKEB_LSCALE, + 'skebnorm': SKEBNORM, + 'skeb_tau': SKEB_TSCALE, + 'skebint': SKEB_INT, + 'skeb_vdof': SKEB_VDOF + }) + + if DO_SPP or DO_LSM_SPP: + nam_stochy_dict.update({ + 'new_lscale': NEW_LSCALE + }) + + settings['nam_stochy'] = nam_stochy_dict + # + # Add the relevant SPP namelist variables to "settings" when running with + # SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. + # + nam_sppperts_dict = {} + if DO_SPP: + nam_sppperts_dict = { + 'iseed_spp': ISEED_SPP, + 'spp_lscale': SPP_LSCALE, + 'spp_prt_list': SPP_MAG_LIST, + 'spp_sigtop1': SPP_SIGTOP1, + 'spp_sigtop2': SPP_SIGTOP2, + 'spp_stddev_cutoff': SPP_STDDEV_CUTOFF, + 'spp_tau': SPP_TSCALE, + 'spp_var_list': SPP_VAR_LIST + } + + settings['nam_sppperts'] = nam_sppperts_dict + # + # Add the relevant LSM SPP namelist variables to "settings" when running with + # LSM SPP turned on. + # + nam_sfcperts_dict = {} + if DO_LSM_SPP: + nam_sfcperts_dict = { + 'lndp_type': LNDP_TYPE, + 'lndp_tau': LSM_SPP_TSCALE, + 'lndp_lscale': LSM_SPP_LSCALE, + 'iseed_lndp': ISEED_LSM_SPP, + 'lndp_var_list': LSM_SPP_VAR_LIST, + 'lndp_prt_list': LSM_SPP_MAG_LIST + } + + settings['nam_sfcperts'] = nam_sfcperts_dict + + settings_str = cfg_to_yaml_str(settings) + + print_info_msg(dedent(f''' + The variable \"settings\" specifying values of the weather model's + namelist variables has been set as follows: + + settings =\n''') + settings_str, verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # Call the set_namelist.py script to create a new FV3 namelist file (full + # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as + # the base (i.e. starting) namelist file, with physics-suite-dependent + # modifications to the base file specified in the yaml configuration file + # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), + # and with additional physics-suite-independent modificaitons specified + # in the variable "settings" set above. + # + #----------------------------------------------------------------------- + # + try: + set_namelist(["-q", "-n", FV3_NML_BASE_SUITE_FP, "-c", FV3_NML_YAML_CONFIG_FP, + CCPP_PHYS_SUITE, "-u", settings_str, "-o", FV3_NML_FP]) + except: + print_err_msg_exit(f''' + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = \"{FV3_NML_BASE_SUITE_FP}\" + Full path to yaml configuration file for various physics suites: + FV3_NML_YAML_CONFIG_FP = \"{FV3_NML_YAML_CONFIG_FP}\" + Physics suite to extract from yaml configuration file: + CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" + Full path to output namelist file: + FV3_NML_FP = \"{FV3_NML_FP}\" + Namelist settings specified on command line: + settings = + {settings_str}''') + # + # If not running the MAKE_GRID_TN task (which implies the workflow will + # use pregenerated grid files), set the namelist variables specifying + # the paths to surface climatology files. These files are located in + # (or have symlinks that point to them) in the FIXLAM directory. + # + # Note that if running the MAKE_GRID_TN task, this action usually cannot + # be performed here but must be performed in that task because the names + # of the surface climatology files depend on the CRES parameter (which is + # the C-resolution of the grid), and this parameter is in most workflow + # configurations is not known until the grid is created. + # + if not RUN_TASK_MAKE_GRID: + + set_FV3nml_sfc_climo_filenames() + # + #----------------------------------------------------------------------- + # + # To have a record of how this experiment/workflow was generated, copy + # the experiment/workflow configuration file to the experiment directo- + # ry. + # + #----------------------------------------------------------------------- + # + cp_vrfy(os.path.join(USHDIR,EXPT_CONFIG_FN), EXPTDIR) + # + #----------------------------------------------------------------------- + # + # For convenience, print out the commands that need to be issued on the + # command line in order to launch the workflow and to check its status. + # Also, print out the line that should be placed in the user's cron table + # in order for the workflow to be continually resubmitted. + # + #----------------------------------------------------------------------- + # + if WORKFLOW_MANAGER == "rocoto": + wflow_db_fn=f"{os.path.splitext(WFLOW_XML_FN)[0]}.db" + rocotorun_cmd=f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + rocotostat_cmd=f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + + print_info_msg(f''' + ======================================================================== + ======================================================================== + + Experiment generation completed. The experiment directory is: + + EXPTDIR=\"{EXPTDIR}\" + + ======================================================================== + ======================================================================== + ''') + # + #----------------------------------------------------------------------- + # + # If rocoto is required, print instructions on how to load and use it + # + #----------------------------------------------------------------------- + # + if WORKFLOW_MANAGER == "rocoto": + + print_info_msg(f''' + To launch the workflow, first ensure that you have a compatible version + of rocoto available. For most pre-configured platforms, rocoto can be + loaded via a module: + + > module load rocoto + + For more details on rocoto, see the User's Guide. + + To launch the workflow, first ensure that you have a compatible version + of rocoto loaded. For example, to load version 1.3.1 of rocoto, use + + > module load rocoto/1.3.1 + + (This version has been tested on hera; later versions may also work but + have not been tested.) + + To launch the workflow, change location to the experiment directory + (EXPTDIR) and issue the rocotrun command, as follows: + + > cd {EXPTDIR} + > {rocotorun_cmd} + + To check on the status of the workflow, issue the rocotostat command + (also from the experiment directory): + + > {rocotostat_cmd} + + Note that: + + 1) The rocotorun command must be issued after the completion of each + task in the workflow in order for the workflow to submit the next + task(s) to the queue. + + 2) In order for the output of the rocotostat command to be up-to-date, + the rocotorun command must be issued immediately before issuing the + rocotostat command. + + For automatic resubmission of the workflow (say every 3 minutes), the + following line can be added to the user's crontab (use \"crontab -e\" to + edit the cron table): + + */3 * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" + ''') + # + # If necessary, run the NOMADS script to source external model data. + # + if NOMADS: + print("Getting NOMADS online data") + print(f"NOMADS_file_type= {NOMADS_file_type}") + cd_vrfy(EXPTDIR) + NOMADS_script = os.path.join(USHDIR, "NOMADS_get_extrn_mdl_files.h") + run_command(f'''{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,True)} \ + {CYCL_HRS} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}''') + + +# +#----------------------------------------------------------------------- +# +# Start of the script that will call the experiment/workflow generation +# function defined above. +# +#----------------------------------------------------------------------- +# +if __name__ == "__main__": + # + #----------------------------------------------------------------------- + # + # Set directories. + # + #----------------------------------------------------------------------- + # + ushdir=os.path.dirname(os.path.abspath(__file__)) + # + # Set the name of and full path to the temporary file in which we will + # save some experiment/workflow variables. The need for this temporary + # file is explained below. + # + tmp_fn="tmp" + tmp_fp=os.path.join(ushdir, tmp_fn) + rm_vrfy("-f",tmp_fp) + # + # Set the name of and full path to the log file in which the output from + # the experiment/workflow generation function will be saved. + # + log_fn="log.generate_FV3LAM_wflow" + log_fp=os.path.join(ushdir, log_fn) + rm_vrfy("-f",log_fp) + # + # Call the generate_FV3LAM_wflow function defined above to generate the + # experiment/workflow. Note that we pipe the output of the function + # (and possibly other commands) to the "tee" command in order to be able + # to both save it to a file and print it out to the screen (stdout). + # The piping causes the call to the function (and the other commands + # grouped with it using the curly braces, { ... }) to be executed in a + # subshell. As a result, the experiment/workflow variables that the + # function sets are not available outside of the grouping, i.e. they are + # not available at and after the call to "tee". Since some of these va- + # riables are needed after the call to "tee" below, we save them in a + # temporary file and read them in outside the subshell later below. + # + def workflow_func(): + retval=1 + generate_FV3LAM_wflow() + retval=0 + run_command(f'''echo "{EXPTDIR}" >> "{tmp_fp}"''') + run_command(f'''echo "{retval}" >> "{tmp_fp}"''') + + # create tee functionality + tee = subprocess.Popen(["tee", log_fp], stdin=subprocess.PIPE) + os.dup2(tee.stdin.fileno(), sys.stdout.fileno()) + os.dup2(tee.stdin.fileno(), sys.stderr.fileno()) + + #create workflow process + p = Process(target=workflow_func) + p.start() + p.join() + + # + # Read in experiment/workflow variables needed later below from the tem- + # porary file created in the subshell above containing the call to the + # generate_FV3LAM_wflow function. These variables are not directly + # available here because the call to generate_FV3LAM_wflow above takes + # place in a subshell (due to the fact that we are then piping its out- + # put to the "tee" command). Then remove the temporary file. + # + (_,exptdir,_)=run_command(f'''sed "1q;d" "{tmp_fp}"''') + (_,retval,_)=run_command(f''' sed "2q;d" "{tmp_fp}"''') + if retval: + retval = int(retval) + else: + retval = 1 + rm_vrfy(tmp_fp) + # + # If the call to the generate_FV3LAM_wflow function above was success- + # ful, move the log file in which the "tee" command saved the output of + # the function to the experiment directory. + # + if retval == 0: + mv_vrfy(log_fp,exptdir) + # + # If the call to the generate_FV3LAM_wflow function above was not suc- + # cessful, print out an error message and exit with a nonzero return + # code. + # + else: + print_err_msg_exit(f''' + Experiment generation failed. Check the log file from the ex- + periment/workflow generation script in the file specified by log_fp: + log_fp = \"{log_fp}\" + Stopping.''') + diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py new file mode 100644 index 0000000000..8316e7c7de --- /dev/null +++ b/ush/get_crontab_contents.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import os +import unittest +from datetime import datetime + +from python_utils import import_vars, set_env_var, print_input_args, \ + run_command, define_macos_utilities, check_var_valid_value +from constants import valid_vals_BOOLEAN + +def get_crontab_contents(called_from_cron): + """ + #----------------------------------------------------------------------- + # + # This function returns the contents of the user's + # cron table as well as the command to use to manipulate the cron table + # (i.e. the "crontab" command, but on some platforms the version or + # location of this may change depending on other circumstances, e.g. on + # Cheyenne, this depends on whether a script that wants to call "crontab" + # is itself being called from a cron job). Arguments are as follows: + # + # called_from_cron: + # Boolean flag that specifies whether this function (and the scripts or + # functions that are calling it) are called as part of a cron job. Must + # be set to "TRUE" or "FALSE". + # + # outvarname_crontab_cmd: + # Name of the output variable that will contain the command to issue for + # the system "crontab" command. + # + # outvarname_crontab_contents: + # Name of the output variable that will contain the contents of the + # user's cron table. + # + #----------------------------------------------------------------------- + """ + + print_input_args(locals()) + + #import all env vars + IMPORTS = ["MACHINE"] + import_vars(env_vars=IMPORTS) + + # + # Make sure called_from_cron is set to a valid value. + # + check_var_valid_value(called_from_cron, valid_vals_BOOLEAN) + + if MACHINE == "WCOSS_DELL_P3": + __crontab_cmd__="" + (_,__crontab_contents__,_)=run_command(f'''cat "/u/{USER}/cron/mycrontab"''') + else: + __crontab_cmd__="crontab" + # + # On Cheyenne, simply typing "crontab" will launch the crontab command + # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized + # version of crontab that will work if called from scripts that are + # themselves being called as cron jobs. In that case, we must instead + # call the system version of crontab at /usr/bin/crontab. + # + if MACHINE == "CHEYENNE": + if called_from_cron: + __crontab_cmd__="/usr/bin/crontab" + (_,__crontab_contents__,_)=run_command(f'''{__crontab_cmd__} -l''') + # + # On Cheyenne, the output of the "crontab -l" command contains a 3-line + # header (comments) at the top that is not actually part of the user's + # cron table. This needs to be removed to avoid adding an unnecessary + # copy of this header to the user's cron table. + # + if MACHINE == "CHEYENNE": + (_,__crontab_contents__,_)=run_command(f'''printf "%s" "{__crontab_contents__}" | tail -n +4 ''') + + return __crontab_cmd__, __crontab_contents__ + +class Testing(unittest.TestCase): + def test_get_crontab_contents(self): + crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=True) + self.assertEqual(crontab_cmd, "crontab") + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG','FALSE') + set_env_var('MACHINE', 'HERA') diff --git a/ush/link_fix.py b/ush/link_fix.py new file mode 100644 index 0000000000..9788a4ad47 --- /dev/null +++ b/ush/link_fix.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 + +import unittest +import os +import glob + +from python_utils import import_vars, set_env_var, print_input_args, \ + print_info_msg, print_err_msg_exit, create_symlink_to_file, \ + define_macos_utilities, check_var_valid_value, \ + cd_vrfy, mkdir_vrfy, find_pattern_in_str + +def link_fix(verbose, file_group): + """ This file defines a function that ... + Args: + verbose: True or False + file_group: could be on of ["grid", "orog", "sfc_climo"] + Returns: + a string: resolution + """ + + print_input_args(locals()) + + valid_vals_file_group=["grid", "orog", "sfc_climo"] + check_var_valid_value(file_group, valid_vals_file_group) + + #import all environement variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # Create symlinks in the FIXLAM directory pointing to the grid files. + # These symlinks are needed by the make_orog, make_sfc_climo, make_ic, + # make_lbc, and/or run_fcst tasks. + # + # Note that we check that each target file exists before attempting to + # create symlinks. This is because the "ln" command will create sym- + # links to non-existent targets without returning with a nonzero exit + # code. + # + #----------------------------------------------------------------------- + # + print_info_msg(f'Creating links in the FIXLAM directory to the grid files...', + verbose=verbose) + # + #----------------------------------------------------------------------- + # + # Create globbing patterns for grid, orography, and surface climatology + # files. + # + # + # For grid files (i.e. file_group set to "grid"), symlinks are created + # in the FIXLAM directory to files (of the same names) in the GRID_DIR. + # These symlinks/files and the reason each is needed is listed below: + # + # 1) "C*.mosaic.halo${NHW}.nc" + # This mosaic file for the wide-halo grid (i.e. the grid with a ${NHW}- + # cell-wide halo) is needed as an input to the orography filtering + # executable in the orography generation task. The filtering code + # extracts from this mosaic file the name of the file containing the + # grid on which it will generate filtered topography. Note that the + # orography generation and filtering are both performed on the wide- + # halo grid. The filtered orography file on the wide-halo grid is then + # shaved down to obtain the filtered orography files with ${NH3}- and + # ${NH4}-cell-wide halos. + # + # The raw orography generation step in the make_orog task requires the + # following symlinks/files: + # + # a) C*.mosaic.halo${NHW}.nc + # The script for the make_orog task extracts the name of the grid + # file from this mosaic file; this name should be + # "C*.grid.tile${TILE_RGNL}.halo${NHW}.nc". + # + # b) C*.grid.tile${TILE_RGNL}.halo${NHW}.nc + # This is the + # The script for the make_orog task passes the name of the grid + # file (extracted above from the mosaic file) to the orography + # generation executable. The executable then + # reads in this grid file and generates a raw orography + # file on the grid. The raw orography file is initially renamed "out.oro.nc", + # but for clarity, it is then renamed "C*.raw_orog.tile${TILE_RGNL}.halo${NHW}.nc". + # + # c) The fixed files thirty.second.antarctic.new.bin, landcover30.fixed, + # and gmted2010.30sec.int. + # + # The orography filtering step in the make_orog task requires the + # following symlinks/files: + # + # a) C*.mosaic.halo${NHW}.nc + # This is the mosaic file for the wide-halo grid. The orography + # filtering executable extracts from this file the name of the grid + # file containing the wide-halo grid (which should be + # "${CRES}.grid.tile${TILE_RGNL}.halo${NHW}.nc"). The executable then + # looks for this grid file IN THE DIRECTORY IN WHICH IT IS RUNNING. + # Thus, before running the executable, the script creates a symlink in this run directory that + # points to the location of the actual wide-halo grid file. + # + # b) C*.raw_orog.tile${TILE_RGNL}.halo${NHW}.nc + # This is the raw orography file on the wide-halo grid. The script + # for the make_orog task copies this file to a new file named + # "C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc" that will be + # used as input to the orography filtering executable. The executable + # will then overwrite the contents of this file with the filtered orography. + # Thus, the output of the orography filtering executable will be + # the file C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc. + # + # The shaving step in the make_orog task requires the following: + # + # a) C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc + # This is the filtered orography file on the wide-halo grid. + # This gets shaved down to two different files: + # + # i) ${CRES}.oro_data.tile${TILE_RGNL}.halo${NH0}.nc + # This is the filtered orography file on the halo-0 grid. + # + # ii) ${CRES}.oro_data.tile${TILE_RGNL}.halo${NH4}.nc + # This is the filtered orography file on the halo-4 grid. + # + # Note that the file names of the shaved files differ from that of + # the initial unshaved file on the wide-halo grid in that the field + # after ${CRES} is now "oro_data" (not "filtered_orog") to comply + # with the naming convention used more generally. + # + # 2) "C*.mosaic.halo${NH4}.nc" + # This mosaic file for the grid with a 4-cell-wide halo is needed as + # an input to the surface climatology generation executable. The + # surface climatology generation code reads from this file the number + # of tiles (which should be 1 for a regional grid) and the tile names. + # More importantly, using the ESMF function ESMF_GridCreateMosaic(), + # it creates a data object of type esmf_grid; the grid information + # in this object is obtained from the grid file specified in the mosaic + # file, which should be "C*.grid.tile${TILE_RGNL}.halo${NH4}.nc". The + # dimensions specified in this grid file must match the ones specified + # in the (filtered) orography file "C*.oro_data.tile${TILE_RGNL}.halo${NH4}.nc" + # that is also an input to the surface climatology generation executable. + # If they do not, then the executable will crash with an ESMF library + # error (something like "Arguments are incompatible"). + # + # Thus, for the make_sfc_climo task, the following symlinks/files must + # exist: + # a) "C*.mosaic.halo${NH4}.nc" + # b) "C*.grid.tile${TILE_RGNL}.halo${NH4}.nc" + # c) "C*.oro_data.tile${TILE_RGNL}.halo${NH4}.nc" + # + # 3) + # + # + #----------------------------------------------------------------------- + # + # + if file_group == "grid": + fns=[ + f"C*{DOT_OR_USCORE}mosaic.halo{NHW}.nc", + f"C*{DOT_OR_USCORE}mosaic.halo{NH4}.nc", + f"C*{DOT_OR_USCORE}mosaic.halo{NH3}.nc", + f"C*{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NHW}.nc", + f"C*{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH3}.nc", + f"C*{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH4}.nc" + ] + fps=[ os.path.join(GRID_DIR,itm) for itm in fns] + run_task=f"{RUN_TASK_MAKE_GRID}" + # + elif file_group == "orog": + fns=[ + f"C*{DOT_OR_USCORE}oro_data.tile{TILE_RGNL}.halo{NH0}.nc", + f"C*{DOT_OR_USCORE}oro_data.tile{TILE_RGNL}.halo{NH4}.nc" + ] + if CCPP_PHYS_SUITE == "FV3_HRRR": + fns+=[ + f"C*{DOT_OR_USCORE}oro_data_ss.tile{TILE_RGNL}.halo{NH0}.nc", + f"C*{DOT_OR_USCORE}oro_data_ls.tile{TILE_RGNL}.halo{NH0}.nc", + ] + fps=[ os.path.join(OROG_DIR,itm) for itm in fns] + run_task=f"{RUN_TASK_MAKE_OROG}" + # + # The following list of symlinks (which have the same names as their + # target files) need to be created made in order for the make_ics and + # make_lbcs tasks (i.e. tasks involving chgres_cube) to work. + # + elif file_group == "sfc_climo": + num_fields=len(SFC_CLIMO_FIELDS) + fns=[None] * (2 * num_fields) + for i in range(num_fields): + ii=2*i + fns[ii]=f"C*.{SFC_CLIMO_FIELDS[i]}.tile{TILE_RGNL}.halo{NH0}.nc" + fns[ii+1]=f"C*.{SFC_CLIMO_FIELDS[i]}.tile{TILE_RGNL}.halo{NH4}.nc" + fps=[ os.path.join(SFC_CLIMO_DIR,itm) for itm in fns] + run_task=f"{RUN_TASK_MAKE_SFC_CLIMO}" + # + + # + #----------------------------------------------------------------------- + # + # Find all files matching the globbing patterns and make sure that they + # all have the same resolution (an integer) in their names. + # + #----------------------------------------------------------------------- + # + i=0 + res_prev="" + res="" + fp_prev="" + + for pattern in fps: + files = glob.glob(pattern) + for fp in files: + + fn = os.path.basename(fp) + + regex_search = "^C([0-9]*).*" + res = find_pattern_in_str(regex_search, fn) + if res is None: + print_err_msg_exit(f''' + The resolution could not be extracted from the current file's name. The + full path to the file (fp) is: + fp = \"{fp}\" + This may be because fp contains the * globbing character, which would + imply that no files were found that match the globbing pattern specified + in fp.''') + else: + res = res[0] + + if ( i > 0 ) and ( res != res_prev ): + print_err_msg_exit(f''' + The resolutions (as obtained from the file names) of the previous and + current file (fp_prev and fp, respectively) are different: + fp_prev = \"{fp_prev}\" + fp = \"{fp}\" + Please ensure that all files have the same resolution.''') + + i=i+1 + fp_prev=f"{fp}" + res_prev=res + # + #----------------------------------------------------------------------- + # + # Replace the * globbing character in the set of globbing patterns with + # the resolution. This will result in a set of (full paths to) specific + # files. + # + #----------------------------------------------------------------------- + # + fps=[ itm.replace('*',res) for itm in fps] + # + #----------------------------------------------------------------------- + # + # In creating the various symlinks below, it is convenient to work in + # the FIXLAM directory. We will change directory back to the original + # later below. + # + #----------------------------------------------------------------------- + # + SAVE_DIR=os.getcwd() + cd_vrfy(FIXLAM) + # + #----------------------------------------------------------------------- + # + # Use the set of full file paths generated above as the link targets to + # create symlinks to these files in the FIXLAM directory. + # + #----------------------------------------------------------------------- + # + # If the task in consideration (which will be one of the pre-processing + # tasks MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN) was run, then + # the target files will be located under the experiment directory. In + # this case, we use relative symlinks in order the experiment directory + # more portable and the symlinks more readable. However, if the task + # was not run, then pregenerated grid, orography, or surface climatology + # files will be used, and those will be located in an arbitrary directory + # (specified by the user) that is somwehere outside the experiment + # directory. Thus, in this case, there isn't really an advantage to using + # relative symlinks, so we use symlinks with absolute paths. + # + if run_task: + relative_link_flag=True + else: + relative_link_flag=False + + for fp in fps: + fn=os.path.basename(fp) + create_symlink_to_file(fp,fn,relative_link_flag) + # + #----------------------------------------------------------------------- + # + # Set the C-resolution based on the resolution appearing in the file + # names. + # + #----------------------------------------------------------------------- + # + cres=f"C{res}" + # + #----------------------------------------------------------------------- + # + # If considering grid files, create a symlink to the halo4 grid file + # that does not contain the halo size in its name. This is needed by + # the tasks that generate the initial and lateral boundary condition + # files. + # + #----------------------------------------------------------------------- + # + if file_group == "grid": + + target=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH4}.nc" + symlink=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.nc" + create_symlink_to_file(target,symlink,True) + # + # The surface climatology file generation code looks for a grid file + # having a name of the form "C${GFDLgrid_RES}_grid.tile7.halo4.nc" (i.e. + # the C-resolution used in the name of this file is the number of grid + # points per horizontal direction per tile, just like in the global model). + # Thus, if we are running the MAKE_SFC_CLIMO_TN task, if the grid is of + # GFDLgrid type, and if we are not using GFDLgrid_RES in filenames (i.e. + # we are using the equivalent global uniform grid resolution instead), + # then create a link whose name uses the GFDLgrid_RES that points to the + # link whose name uses the equivalent global uniform resolution. + # + if RUN_TASK_MAKE_SFC_CLIMO and \ + GRID_GEN_METHOD == "GFDLgrid" and \ + not GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: + target=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH4}.nc" + symlink=f"C{GFDLgrid_RES}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.nc" + create_symlink_to_file(target,symlink,relative) + # + #----------------------------------------------------------------------- + # + # If considering surface climatology files, create symlinks to the surface + # climatology files that do not contain the halo size in their names. + # These are needed by the task that generates the initial condition files. + # + #----------------------------------------------------------------------- + # + if file_group == "sfc_climo": + + tmp=[ f"{cres}.{itm}" for itm in SFC_CLIMO_FIELDS] + fns_sfc_climo_with_halo_in_fn=[ f"{itm}.tile{TILE_RGNL}.halo{NH4}.nc" for itm in tmp] + fns_sfc_climo_no_halo_in_fn=[ f"{itm}.tile{TILE_RGNL}.nc" for itm in tmp] + + for i in range(num_fields): + target=f"{fns_sfc_climo_with_halo_in_fn[i]}" + symlink=f"{fns_sfc_climo_no_halo_in_fn[i]}" + create_symlink_to_file(target, symlink, True) + # + # In order to be able to specify the surface climatology file names in + # the forecast model's namelist file, in the FIXLAM directory a symlink + # must be created for each surface climatology field that has "tile1" in + # its name (and no "halo") and which points to the corresponding "tile7.halo0" + # file. + # + tmp=[ f"{cres}.{itm}" for itm in SFC_CLIMO_FIELDS ] + fns_sfc_climo_tile7_halo0_in_fn=[ f"{itm}.tile{TILE_RGNL}.halo{NH0}.nc" for itm in tmp ] + fns_sfc_climo_tile1_no_halo_in_fn=[ f"{itm}.tile1.nc" for itm in tmp ] + + for i in range(num_fields): + target=f"{fns_sfc_climo_tile7_halo0_in_fn[i]}" + symlink=f"{fns_sfc_climo_tile1_no_halo_in_fn[i]}" + create_symlink_to_file(target,symlink,True) + # + #----------------------------------------------------------------------- + # + # Change directory back to original one. + # + #----------------------------------------------------------------------- + # + cd_vrfy(SAVE_DIR) + + return res + +class Testing(unittest.TestCase): + def test_link_fix(self): + res = link_fix(verbose=True, file_group="grid") + self.assertTrue( res == "3357") + def setUp(self): + define_macos_utilities() + TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data"); + FIXLAM = os.path.join(TEST_DIR, "expt", "fix_lam") + mkdir_vrfy("-p",FIXLAM) + set_env_var("FIXLAM",FIXLAM) + set_env_var("DOT_OR_USCORE","_") + set_env_var("TILE_RGNL",7) + set_env_var("NH0",0) + set_env_var("NHW",6) + set_env_var("NH4",4) + set_env_var("NH3",3) + set_env_var("GRID_DIR",TEST_DIR + os.sep + "RRFS_CONUS_3km") + set_env_var("RUN_TASK_MAKE_GRID","FALSE") + set_env_var("OROG_DIR",TEST_DIR + os.sep + "RRFS_CONUS_3km") + set_env_var("RUN_TASK_MAKE_OROG","FALSE") + set_env_var("SFC_CLIMO_DIR",TEST_DIR + os.sep + "RRFS_CONUS_3km") + set_env_var("RUN_TASK_MAKE_SFC_CLIMO","FALSE") + set_env_var("CCPP_PHYS_SUITE","FV3_GSD_SAR") diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml new file mode 100644 index 0000000000..39701ca62d --- /dev/null +++ b/ush/predef_grid_params.yaml @@ -0,0 +1,866 @@ +# +#----------------------------------------------------------------------- +# +# Set grid and other parameters according to the value of the predefined +# domain (PREDEF_GRID_NAME). Note that the code will enter this script +# only if PREDEF_GRID_NAME has a valid (and non-empty) value. +# +#################### +# The following comments need to be updated: +#################### +# +# 1) Reset the experiment title (expt_title). +# 2) Reset the grid parameters. +# 3) If the write component is to be used (i.e. QUILTING is set to +# "TRUE") and the variable WRTCMP_PARAMS_TMPL_FN containing the name +# of the write-component template file is unset or empty, set that +# filename variable to the appropriate preexisting template file. +# +# For the predefined domains, we determine the starting and ending indi- +# ces of the regional grid within tile 6 by specifying margins (in units +# of number of cells on tile 6) between the boundary of tile 6 and that +# of the regional grid (tile 7) along the left, right, bottom, and top +# portions of these boundaries. Note that we do not use "west", "east", +# "south", and "north" here because the tiles aren't necessarily orient- +# ed such that the left boundary segment corresponds to the west edge, +# etc. The widths of these margins (in units of number of cells on tile +# 6) are specified via the parameters +# +# num_margin_cells_T6_left +# num_margin_cells_T6_right +# num_margin_cells_T6_bottom +# num_margin_cells_T6_top +# +# where the "_T6" in these names is used to indicate that the cell count +# is on tile 6, not tile 7. +# +# Note that we must make the margins wide enough (by making the above +# four parameters large enough) such that a region of halo cells around +# the boundary of the regional grid fits into the margins, i.e. such +# that the halo does not overrun the boundary of tile 6. (The halo is +# added later in another script; its function is to feed in boundary +# conditions to the regional grid.) Currently, a halo of 5 regional +# grid cells is used around the regional grid. Setting num_margin_- +# cells_T6_... to at least 10 leaves enough room for this halo. +# +#----------------------------------------------------------------------- +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~25km cells. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUS_25km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 25000.0 + ESGgrid_DELY: 25000.0 + ESGgrid_NX: 219 + ESGgrid_NY: 131 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 5 + LAYOUT_Y: 2 + BLOCKSIZE: 40 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 2 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 217 + WRTCMP_ny: 128 + WRTCMP_lon_lwr_left: -122.719528 + WRTCMP_lat_lwr_left: 21.138123 + WRTCMP_dx: 25000.0 + WRTCMP_dy: 25000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~25km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_25km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 25000.0 + ESGgrid_DELY: 25000.0 + ESGgrid_NX: 202 + ESGgrid_NY: 116 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 5 + LAYOUT_Y: 2 + BLOCKSIZE: 40 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 2 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 199 + WRTCMP_ny: 111 + WRTCMP_lon_lwr_left: -121.23349066 + WRTCMP_lat_lwr_left: 23.41731593 + WRTCMP_dx: 25000.0 + WRTCMP_dy: 25000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~13km cells. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUS_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 420 + ESGgrid_NY: 252 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 45 + LAYOUT_X: 16 + LAYOUT_Y: 10 + BLOCKSIZE: 32 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 10 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 416 + WRTCMP_ny: 245 + WRTCMP_lon_lwr_left: -122.719528 + WRTCMP_lat_lwr_left: 21.138123 + WRTCMP_dx: 13000.0 + WRTCMP_dy: 13000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~13km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 396 + ESGgrid_NY: 232 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 45 + LAYOUT_X: 16 + LAYOUT_Y: 10 + BLOCKSIZE: 32 + # if QUILTING = TRUE + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 16 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 393 + WRTCMP_ny: 225 + WRTCMP_lon_lwr_left: -121.70231097 + WRTCMP_lat_lwr_left: 22.57417972 + WRTCMP_dx: 13000.0 + WRTCMP_dy: 13000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~13km cells. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUS_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 396 + ESGgrid_NY: 232 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 45 + LAYOUT_X: 16 + LAYOUT_Y: 10 + BLOCKSIZE: 32 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 10 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 393 + WRTCMP_ny: 225 + WRTCMP_lon_lwr_left: -121.70231097 + WRTCMP_lat_lwr_left: 22.57417972 + WRTCMP_dx: 13000.0 + WRTCMP_dy: 13000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~3km cells. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUS_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 1820 + ESGgrid_NY: 1092 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 36 + LAYOUT_X: 28 + LAYOUT_Y: 28 + BLOCKSIZE: 29 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 28 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 1799 + WRTCMP_ny: 1059 + WRTCMP_lon_lwr_left: -122.719528 + WRTCMP_lat_lwr_left: 21.138123 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS CONUS domain with ~3km cells that can be initialized from the HRRR. +# +#----------------------------------------------------------------------- +# +"RRFS_CONUScompact_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 1748 + ESGgrid_NY: 1038 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 30 + LAYOUT_Y: 16 + BLOCKSIZE: 32 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 16 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 1746 + WRTCMP_ny: 1014 + WRTCMP_lon_lwr_left: -122.17364391 + WRTCMP_lat_lwr_left: 21.88588562 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS SUBCONUS domain with ~3km cells. +# +#----------------------------------------------------------------------- +# +"RRFS_SUBCONUS_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 35.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 840 + ESGgrid_NY: 600 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 30 + LAYOUT_Y: 24 + BLOCKSIZE: 35 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 24 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 35.0 + WRTCMP_stdlat1: 35.0 + WRTCMP_stdlat2: 35.0 + WRTCMP_nx: 837 + WRTCMP_ny: 595 + WRTCMP_lon_lwr_left: -109.97410429 + WRTCMP_lat_lwr_left: 26.31459843 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# A subconus domain over Indianapolis, Indiana with ~3km cells. This is +# mostly for testing on a 3km grid with a much small number of cells than +# on the full CONUS. +# +#----------------------------------------------------------------------- +# +"SUBCONUS_Ind_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -86.16 + ESGgrid_LAT_CTR: 39.77 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 200 + ESGgrid_NY: 200 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 5 + LAYOUT_Y: 5 + BLOCKSIZE: 40 + #if QUILTING : True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 5 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -86.16 + WRTCMP_cen_lat: 39.77 + WRTCMP_stdlat1: 39.77 + WRTCMP_stdlat2: 39.77 + WRTCMP_nx: 197 + WRTCMP_ny: 195 + WRTCMP_lon_lwr_left: -89.47120417 + WRTCMP_lat_lwr_left: 37.07809642 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS Alaska domain with ~13km cells. +# +# Note: +# This grid has not been thoroughly tested (as of 20201027). +# +#----------------------------------------------------------------------- +# +"RRFS_AK_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -161.5 + ESGgrid_LAT_CTR: 63.0 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 320 + ESGgrid_NY: 240 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 10 + LAYOUT_X: 16 + LAYOUT_Y: 12 + BLOCKSIZE: 40 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 12 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -161.5 + WRTCMP_cen_lat: 63.0 + WRTCMP_stdlat1: 63.0 + WRTCMP_stdlat2: 63.0 + WRTCMP_nx: 318 + WRTCMP_ny: 234 + WRTCMP_lon_lwr_left: 172.23339164 + WRTCMP_lat_lwr_left: 45.77691870 + WRTCMP_dx: 13000.0 + WRTCMP_dy: 13000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS Alaska domain with ~3km cells. +# +# Note: +# This grid has not been thoroughly tested (as of 20201027). +# +#----------------------------------------------------------------------- +# +"RRFS_AK_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -161.5 + ESGgrid_LAT_CTR: 63.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 1380 + ESGgrid_NY: 1020 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 10 + LAYOUT_X: 30 + LAYOUT_Y: 17 + BLOCKSIZE: 40 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 17 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -161.5 + WRTCMP_cen_lat: 63.0 + WRTCMP_stdlat1: 63.0 + WRTCMP_stdlat2: 63.0 + WRTCMP_nx: 1379 + WRTCMP_ny: 1003 + WRTCMP_lon_lwr_left: -187.89737923 + WRTCMP_lat_lwr_left: 45.84576053 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# A CONUS domain of GFDLgrid type with ~25km cells. +# +# Note: +# This grid is larger than the HRRRX domain and thus cannot be initialized +# using the HRRRX. +# +#----------------------------------------------------------------------- +# +"CONUS_25km_GFDLgrid": + GRID_GEN_METHOD: "GFDLgrid" + GFDLgrid_LON_T6_CTR: -97.5 + GFDLgrid_LAT_T6_CTR: 38.5 + GFDLgrid_STRETCH_FAC: 1.4 + GFDLgrid_RES: 96 + GFDLgrid_REFINE_RATIO: 3 + num_margin_cells_T6_left: 12 + GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: 13 + num_margin_cells_T6_right: 12 + GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: 84 + num_margin_cells_T6_bottom: 16 + GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: 17 + num_margin_cells_T6_top: 16 + GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: 80 + GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: True + DT_ATMOS: 225 + LAYOUT_X: 6 + LAYOUT_Y: 4 + BLOCKSIZE: 36 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 4 + WRTCMP_output_grid: "rotated_latlon" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_lon_lwr_left: -24.40085141 + WRTCMP_lat_lwr_left: -19.65624142 + WRTCMP_lon_upr_rght: 24.40085141 + WRTCMP_lat_upr_rght: 19.65624142 + WRTCMP_dlon: 0.22593381 + WRTCMP_dlat: 0.22593381 +# +#----------------------------------------------------------------------- +# +# A CONUS domain of GFDLgrid type with ~3km cells. +# +# Note: +# This grid is larger than the HRRRX domain and thus cannot be initialized +# using the HRRRX. +# +#----------------------------------------------------------------------- +# +"CONUS_3km_GFDLgrid": + GRID_GEN_METHOD: "GFDLgrid" + GFDLgrid_LON_T6_CTR: -97.5 + GFDLgrid_LAT_T6_CTR: 38.5 + GFDLgrid_STRETCH_FAC: 1.5 + GFDLgrid_RES: 768 + GFDLgrid_REFINE_RATIO: 3 + num_margin_cells_T6_left: 69 + GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: 70 + num_margin_cells_T6_right: 69 + GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: 699 + num_margin_cells_T6_bottom: 164 + GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: 165 + num_margin_cells_T6_top: 164 + GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: 604 + GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: True + DT_ATMOS: 18 + LAYOUT_X: 30 + LAYOUT_Y: 22 + BLOCKSIZE: 35 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 22 + WRTCMP_output_grid: "rotated_latlon" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_lon_lwr_left: -25.23144805 + WRTCMP_lat_lwr_left: -15.82130419 + WRTCMP_lon_upr_rght: 25.23144805 + WRTCMP_lat_upr_rght: 15.82130419 + WRTCMP_dlon: 0.02665763 + WRTCMP_dlat: 0.02665763 +# +#----------------------------------------------------------------------- +# +# EMC's Alaska grid. +# +#----------------------------------------------------------------------- +# +"EMC_AK": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -153.0 + ESGgrid_LAT_CTR: 61.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 1344 # Supergrid value 2704 + ESGgrid_NY: 1152 # Supergrid value 2320 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 18 + LAYOUT_X: 28 + LAYOUT_Y: 16 + BLOCKSIZE: 24 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 24 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -153.0 + WRTCMP_cen_lat: 61.0 + WRTCMP_stdlat1: 61.0 + WRTCMP_stdlat2: 61.0 + WRTCMP_nx: 1344 + WRTCMP_ny: 1152 + WRTCMP_lon_lwr_left: -177.0 + WRTCMP_lat_lwr_left: 42.5 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# EMC's Hawaii grid. +# +#----------------------------------------------------------------------- +# +"EMC_HI": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -157.0 + ESGgrid_LAT_CTR: 20.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 432 # Supergrid value 880 + ESGgrid_NY: 360 # Supergrid value 736 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 18 + LAYOUT_X: 8 + LAYOUT_Y: 8 + BLOCKSIZE: 27 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 8 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -157.0 + WRTCMP_cen_lat: 20.0 + WRTCMP_stdlat1: 20.0 + WRTCMP_stdlat2: 20.0 + WRTCMP_nx: 420 + WRTCMP_ny: 348 + WRTCMP_lon_lwr_left: -162.8 + WRTCMP_lat_lwr_left: 15.2 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# EMC's Puerto Rico grid. +# +#----------------------------------------------------------------------- +# +"EMC_PR": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -69.0 + ESGgrid_LAT_CTR: 18.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 576 # Supergrid value 1168 + ESGgrid_NY: 432 # Supergrid value 880 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 18 + LAYOUT_X: 16 + LAYOUT_Y: 8 + BLOCKSIZE: 24 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 24 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -69.0 + WRTCMP_cen_lat: 18.0 + WRTCMP_stdlat1: 18.0 + WRTCMP_stdlat2: 18.0 + WRTCMP_nx: 576 + WRTCMP_ny: 432 + WRTCMP_lon_lwr_left: -77 + WRTCMP_lat_lwr_left: 12 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# EMC's Guam grid. +# +#----------------------------------------------------------------------- +# +"EMC_GU": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: 146.0 + ESGgrid_LAT_CTR: 15.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 432 # Supergrid value 880 + ESGgrid_NY: 360 # Supergrid value 736 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 18 + LAYOUT_X: 16 + LAYOUT_Y: 12 + BLOCKSIZE: 27 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 24 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: 146.0 + WRTCMP_cen_lat: 15.0 + WRTCMP_stdlat1: 15.0 + WRTCMP_stdlat2: 15.0 + WRTCMP_nx: 420 + WRTCMP_ny: 348 + WRTCMP_lon_lwr_left: 140 + WRTCMP_lat_lwr_left: 10 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# Emulation of the HAFS v0.A grid at 25 km. +# +#----------------------------------------------------------------------- +# +"GSL_HAFSV0.A_25km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -62.0 + ESGgrid_LAT_CTR: 22.0 + ESGgrid_DELX: 25000.0 + ESGgrid_DELY: 25000.0 + ESGgrid_NX: 345 + ESGgrid_NY: 230 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 300 + LAYOUT_X: 5 + LAYOUT_Y: 5 + BLOCKSIZE: 6 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 32 + WRTCMP_output_grid: "regional_latlon" + WRTCMP_cen_lon: -62.0 + WRTCMP_cen_lat: 25.0 + WRTCMP_lon_lwr_left: -114.5 + WRTCMP_lat_lwr_left: -5.0 + WRTCMP_lon_upr_rght: -9.5 + WRTCMP_lat_upr_rght: 55.0 + WRTCMP_dlon: 0.25 + WRTCMP_dlat: 0.25 +# +#----------------------------------------------------------------------- +# +# Emulation of the HAFS v0.A grid at 13 km. +# +#----------------------------------------------------------------------- +# +"GSL_HAFSV0.A_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -62.0 + ESGgrid_LAT_CTR: 22.0 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 665 + ESGgrid_NY: 444 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 180 + LAYOUT_X: 19 + LAYOUT_Y: 12 + BLOCKSIZE: 35 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 32 + WRTCMP_output_grid: "regional_latlon" + WRTCMP_cen_lon: -62.0 + WRTCMP_cen_lat: 25.0 + WRTCMP_lon_lwr_left: -114.5 + WRTCMP_lat_lwr_left: -5.0 + WRTCMP_lon_upr_rght: -9.5 + WRTCMP_lat_upr_rght: 55.0 + WRTCMP_dlon: 0.13 + WRTCMP_dlat: 0.13 +# +#----------------------------------------------------------------------- +# +# Emulation of the HAFS v0.A grid at 3 km. +# +#----------------------------------------------------------------------- +# +"GSL_HAFSV0.A_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -62.0 + ESGgrid_LAT_CTR: 22.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 2880 + ESGgrid_NY: 1920 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 40 + LAYOUT_X: 32 + LAYOUT_Y: 24 + BLOCKSIZE: 32 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 32 + WRTCMP_output_grid: "regional_latlon" + WRTCMP_cen_lon: -62.0 + WRTCMP_cen_lat: 25.0 + WRTCMP_lon_lwr_left: -114.5 + WRTCMP_lat_lwr_left: -5.0 + WRTCMP_lon_upr_rght: -9.5 + WRTCMP_lat_upr_rght: 55.0 + WRTCMP_dlon: 0.03 + WRTCMP_dlat: 0.03 +# +#----------------------------------------------------------------------- +# +# 50-km HRRR Alaska grid. +# +#----------------------------------------------------------------------- +# +"GSD_HRRR_AK_50km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -163.5 + ESGgrid_LAT_CTR: 62.8 + ESGgrid_DELX: 50000.0 + ESGgrid_DELY: 50000.0 + ESGgrid_NX: 74 + ESGgrid_NY: 51 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 600 + LAYOUT_X: 2 + LAYOUT_Y: 3 + BLOCKSIZE: 37 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 1 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -163.5 + WRTCMP_cen_lat: 62.8 + WRTCMP_stdlat1: 62.8 + WRTCMP_stdlat2: 62.8 + WRTCMP_nx: 70 + WRTCMP_ny: 45 + WRTCMP_lon_lwr_left: 172.0 + WRTCMP_lat_lwr_left: 49.0 + WRTCMP_dx: 50000.0 + WRTCMP_dy: 50000.0 +# +#----------------------------------------------------------------------- +# +# Emulation of GSD's RAP domain with ~13km cell size. +# +#----------------------------------------------------------------------- +# +"RRFS_NA_13km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -112.5 + ESGgrid_LAT_CTR: 55.0 + ESGgrid_DELX: 13000.0 + ESGgrid_DELY: 13000.0 + ESGgrid_NX: 912 + ESGgrid_NY: 623 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 50 + LAYOUT_X: 16 + LAYOUT_Y: 16 + BLOCKSIZE: 30 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 16 + WRTCMP_output_grid: "rotated_latlon" + WRTCMP_cen_lon: -113.0 + WRTCMP_cen_lat: 55.0 + WRTCMP_lon_lwr_left: -61.0 + WRTCMP_lat_lwr_left: -37.0 + WRTCMP_lon_upr_rght: 61.0 + WRTCMP_lat_upr_rght: 37.0 + WRTCMP_dlon: 0.1169081 + WRTCMP_dlat: 0.1169081 +# +#----------------------------------------------------------------------- +# +# Future operational RRFS domain with ~3km cell size. +# +#----------------------------------------------------------------------- +# +"RRFS_NA_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -112.5 + ESGgrid_LAT_CTR: 55.0 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 3950 + ESGgrid_NY: 2700 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 36 + LAYOUT_X: 20 # 40 - EMC operational configuration + LAYOUT_Y: 35 # 45 - EMC operational configuration + BLOCKSIZE: 28 + #if QUILTING = True + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 144 + WRTCMP_output_grid: "rotated_latlon" + WRTCMP_cen_lon: -113.0 + WRTCMP_cen_lat: 55.0 + WRTCMP_lon_lwr_left: -61.0 + WRTCMP_lat_lwr_left: -37.0 + WRTCMP_lon_upr_rght: 61.0 + WRTCMP_lat_upr_rght: 37.0 + WRTCMP_dlon: 0.025 + WRTCMP_dlat: 0.025 + diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index 1e4f6feed5..9371488d5c 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -1,4 +1,4 @@ -from .change_case import uppercase, lowercase +from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file from .check_for_preexist_dir_file import check_for_preexist_dir_file from .check_var_valid_value import check_var_valid_value from .count_files import count_files @@ -9,12 +9,14 @@ from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy from .get_charvar_from_netcdf import get_charvar_from_netcdf from .get_elem_inds import get_elem_inds -from .get_manage_externals_config_property import get_manage_externals_config_property from .interpol_to_arbit_CRES import interpol_to_arbit_CRES from .print_input_args import print_input_args from .print_msg import print_info_msg, print_err_msg_exit from .process_args import process_args from .run_command import run_command -from .config_parser import cfg_to_shell_str, cfg_to_yaml_str, yaml_safe_load, \ - load_shell_config, load_config_file - +from .config_parser import load_yaml_config, cfg_to_yaml_str, \ + load_json_config, cfg_to_json_str, \ + load_ini_config, cfg_to_ini_str, get_ini_value, \ + load_shell_config, cfg_to_shell_str, \ + load_config_file +from .xml_parser import load_xml_file, has_tag_with_value diff --git a/ush/python_utils/change_case.py b/ush/python_utils/change_case.py deleted file mode 100644 index 4fb46f94e4..0000000000 --- a/ush/python_utils/change_case.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -def uppercase(str): - """ Function to convert a given string to uppercase - - Args: - str: the string - Return: - Uppercased str - """ - - return str.upper() - - -def lowercase(str): - """ Function to convert a given string to lowercase - - Args: - str: the string - Return: - Lowercase str - """ - - return str.lower() - diff --git a/ush/python_utils/check_for_preexist_dir_file.py b/ush/python_utils/check_for_preexist_dir_file.py index 743f80fcc6..97c709cf0f 100644 --- a/ush/python_utils/check_for_preexist_dir_file.py +++ b/ush/python_utils/check_for_preexist_dir_file.py @@ -24,7 +24,7 @@ def check_for_preexist_dir_file(path, method): rm_vrfy(' -rf ', path) elif method == 'rename': now = datetime.now() - d = now.strftime("_%Y%m%d_%H%M%S") + d = now.strftime("_old_%Y%m%d_%H%M%S") new_path = path + d print_info_msg(f''' Specified directory or file already exists: diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index f7d5a86eed..da7131df26 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -1,15 +1,78 @@ #!/usr/bin/env python3 + +""" +This file provides utilities for processing different configuration file formats. +Supported formats include: + a) YAML + b) JSON + c) SHELL + d) INI + +Typical usage involves first loading the config file, then using the dictionary +returnded by load_config to make queries. +""" + import argparse import yaml +import json import sys import os from textwrap import dedent +import configparser from .environment import list_to_str, str_to_list from .print_msg import print_err_msg_exit from .run_command import run_command +########## +# YAML +########## +def load_yaml_config(config_file): + """ Safe load a yaml file """ + + try: + with open(config_file,'r') as f: + cfg = yaml.safe_load(f) + except yaml.YAMLError as e: + print_err_msg_exit(e) + + return cfg + +def cfg_to_yaml_str(cfg): + """ Get contents of config file as a yaml string """ + + return yaml.dump(cfg, sort_keys=False, default_flow_style=False) + +def join_str(loader, node): + """ Custom tag hangler to join strings """ + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + +yaml.add_constructor('!join_str', join_str, Loader=yaml.SafeLoader) + +########## +# JSON +########## +def load_json_config(config_file): + """ Load json config file """ + + try: + with open(config_file,'r') as f: + cfg = json.load(f) + except: + print_err_msg_exit(e) + + return cfg + +def cfg_to_json_str(cfg): + """ Get contents of config file as a json string """ + + return json.dumps(cfg, sort_keys=False, indent=4) + +########## +# SHELL +########## def load_shell_config(config_file): """ Loads old style shell config files. We source the config script in a subshell and gets the variables it sets @@ -24,10 +87,11 @@ def load_shell_config(config_file): # do a diff to get variables specifically defined/updated in the script # Method sounds brittle but seems to work ok so far code = dedent(f''' #!/bin/bash - (set -o posix; set) > /tmp/t1 + (set -o posix; set) > ./_t1 {{ . {config_file}; set +x; }} &>/dev/null - (set -o posix; set) > /tmp/t2 - diff /tmp/t1 /tmp/t2 | grep "> " | cut -c 3- + (set -o posix; set) > ./_t2 + diff ./_t1 ./_t2 | grep "> " | cut -c 3- + rm -rf ./_t1 ./_t2 ''') (_,config_str,_) = run_command(code) lines = config_str.splitlines() @@ -41,16 +105,16 @@ def load_shell_config(config_file): cfg[k] = v return cfg -def cfg_to_yaml_str(cfg): - """ Get contents of config file as a yaml string """ - - return yaml.dump(cfg, sort_keys=False, default_flow_style=False) - def cfg_to_shell_str(cfg): - """ Get contents of yaml file as shell script string""" + """ Get contents of config file as shell script string""" shell_str = '' for k,v in cfg.items(): + if isinstance(v,dict): + shell_str += f"# [{k}]\n" + shell_str += cfg_to_shell_str(v) + shell_str += "\n" + continue v1 = list_to_str(v) if isinstance(v,list): shell_str += f'{k}={v1}\n' @@ -58,46 +122,85 @@ def cfg_to_shell_str(cfg): shell_str += f"{k}='{v1}'\n" return shell_str -def yaml_safe_load(file_name): - """ Safe load a yaml file """ +########## +# INI +########## +def load_ini_config(config_file): + """ Load a config file with a format similar to Microsoft's INI files""" - try: - with open(file_name,'r') as f: - cfg = yaml.safe_load(f) - except yaml.YAMLError as e: - print_err_msg_exit(e) + if not os.path.exists(config_file): + print_err_msg_exit(f''' + The specified configuration file does not exist: + \"{file_name}\"''') + + config = configparser.ConfigParser() + config.read(config_file) + config_dict = {s:dict(config.items(s)) for s in config.sections()} + return config_dict + +def get_ini_value(config, section, key): + """ Finds the value of a property in a given section""" + + if not section in config: + print_err_msg_exit(f''' + Section not found: + section = \"{section}\" + valid sections = \"{config.keys()}\"''') + else: + return config[section][key] - return cfg + return None -def join(loader, node): - """ Custom tag hangler to join strings """ - seq = loader.construct_sequence(node) - return ''.join([str(i) for i in seq]) +def cfg_to_ini_str(cfg): + """ Get contents of config file as ini string""" -yaml.add_constructor('!join', join, Loader=yaml.SafeLoader) + ini_str = '' + for k,v in cfg.items(): + if isinstance(v,dict): + ini_str += f"[{k}]\n" + ini_str += cfg_to_ini_str(v) + ini_str += "\n" + continue + v1 = list_to_str(v) + if isinstance(v,list): + ini_str += f'{k}={v1}\n' + else: + ini_str += f"{k}='{v1}'\n" + return ini_str +################## +# CONFIG loader +################## def load_config_file(file_name): - """ Choose yaml/shell file based on extension """ + """ Load config file based on file name extension """ + ext = os.path.splitext(file_name)[1][1:] if ext == "sh": return load_shell_config(file_name) + elif ext == "cfg": + return load_ini_config(file_name) + elif ext == "json": + return load_json_config(file_name) else: - return yaml_safe_load(file_name) - + return load_yaml_config(file_name) if __name__ == "__main__": parser = argparse.ArgumentParser(description=\ - 'Prints contents of yaml file as bash argument-value pairs.') + 'Prints contents of config file.') parser.add_argument('--cfg','-c',dest='cfg',required=True, - help='yaml or regular shell config file to parse') + help='config file to parse') parser.add_argument('--output-type','-o',dest='out_type',required=False, - help='output format: "shell": shell format, any other: yaml format ') + help='output format: can be any of ["shell", "yaml", "ini", "json"]') args = parser.parse_args() cfg = load_config_file(args.cfg) if args.out_type == 'shell': print( cfg_to_shell_str(cfg) ) + elif args.out_type == 'ini': + print( cfg_to_ini_str(cfg) ) + elif args.out_type == 'json': + print( cfg_to_json_str(cfg) ) else: print( cfg_to_yaml_str(cfg) ) diff --git a/ush/python_utils/environment.py b/ush/python_utils/environment.py index e98deaad26..25f03b8fd8 100644 --- a/ush/python_utils/environment.py +++ b/ush/python_utils/environment.py @@ -126,7 +126,7 @@ def str_to_list(v): v = v.strip() if not v: return None - if v[0] == '(': + if v[0] == '(' and v[-1] == ')': v = v[1:-1] tokens = shlex.split(v) lst = [] diff --git a/ush/python_utils/get_elem_inds.py b/ush/python_utils/get_elem_inds.py index fd4ea2cee0..08cbfbff70 100644 --- a/ush/python_utils/get_elem_inds.py +++ b/ush/python_utils/get_elem_inds.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from .change_case import lowercase +from .misc import lowercase from .check_var_valid_value import check_var_valid_value def get_elem_inds(arr, match, ret_type): diff --git a/ush/python_utils/get_manage_externals_config_property.py b/ush/python_utils/get_manage_externals_config_property.py deleted file mode 100644 index c537a88244..0000000000 --- a/ush/python_utils/get_manage_externals_config_property.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 - -import os -import configparser - -from .print_msg import print_err_msg_exit - -def get_manage_externals_config_property(externals_cfg_fp, external_name, property_name): - """ - This function searches a specified manage_externals configuration file - and extracts from it the value of the specified property of the external - with the specified name (e.g. the relative path in which the external - has been/will be cloned by the manage_externals utility). - - Args: - - externals_cfg_fp: - The absolute or relative path to the manage_externals configuration - file that will be searched. - - external_name: - The name of the external to search for in the manage_externals confi- - guration file specified by externals_cfg_fp. - - property_name: - The name of the property whose value to obtain (for the external spe- - cified by external_name). - - Returns: - The property value - """ - - if not os.path.exists(externals_cfg_fp): - print_err_msg_exit(f''' - The specified manage_externals configuration file (externals_cfg_fp) - does not exist: - externals_cfg_fp = \"{externals_cfg_fp}\"''') - - config = configparser.ConfigParser() - config.read(externals_cfg_fp) - - if not external_name in config.sections(): - print_err_msg_exit(f''' - In the specified manage_externals configuration file (externals_cfg_fp), - the specified property (property_name) was not found for the the speci- - fied external (external_name): - externals_cfg_fp = \"{externals_cfg_fp}\" - external_name = \"{external_name}\" - property_name = \"{property_name}\"''') - else: - return config[external_name][property_name] - - return None - diff --git a/ush/python_utils/misc.py b/ush/python_utils/misc.py new file mode 100644 index 0000000000..1934ac3d6c --- /dev/null +++ b/ush/python_utils/misc.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import re + +def uppercase(str): + """ Function to convert a given string to uppercase + + Args: + str: the string + Return: + Uppercased str + """ + + return str.upper() + + +def lowercase(str): + """ Function to convert a given string to lowercase + + Args: + str: the string + Return: + Lowercase str + """ + + return str.lower() + +def find_pattern_in_str(pattern, source): + """ Find regex pattern in a string + + Args: + pattern: regex expression + source: string + Return: + A tuple of matched groups or None + """ + pattern = re.compile(pattern) + for match in re.finditer(pattern,source): + return match.groups() + return None + +def find_pattern_in_file(pattern, file_name): + """ Find regex pattern in a file + + Args: + pattern: regex expression + file_name: name of text file + Return: + A tuple of matched groups or None + """ + pattern = re.compile(pattern) + with open(file_name) as f: + for line in f: + for match in re.finditer(pattern,line): + return match.groups() + return None + diff --git a/ush/python_utils/print_input_args.py b/ush/python_utils/print_input_args.py index 4f168a612f..25d979eae9 100644 --- a/ush/python_utils/print_input_args.py +++ b/ush/python_utils/print_input_args.py @@ -4,7 +4,7 @@ import inspect from textwrap import dedent -from .change_case import lowercase +from .misc import lowercase from .print_msg import print_info_msg from .environment import import_vars diff --git a/ush/python_utils/test_python_utils.py b/ush/python_utils/test_python_utils.py index 568de918e5..52defb1c6c 100644 --- a/ush/python_utils/test_python_utils.py +++ b/ush/python_utils/test_python_utils.py @@ -18,9 +18,23 @@ from python_utils import * class Testing(unittest.TestCase): - def test_change_case(self): + def test_misc(self): self.assertEqual( uppercase('upper'), 'UPPER' ) self.assertEqual( lowercase('LOWER'), 'lower' ) + # regex in file + pattern = f'^[ ]*(lsm_ruc)<\/scheme>[ ]*$' + FILE=f"{self.PATH}/../test_data/suite_FV3_GSD_SAR.xml" + match = find_pattern_in_file(pattern, FILE) + self.assertEqual( ("lsm_ruc",), match) + # regex in string + with open(FILE) as f: + content = f.read() + find_pattern_in_str(pattern, content) + self.assertEqual( ("lsm_ruc",), match) + def test_xml_parser(self): + FILE=f"{self.PATH}/../test_data/suite_FV3_GSD_SAR.xml" + tree = load_xml_file(FILE) + self.assertTrue(has_tag_with_value(tree,"scheme","lsm_ruc")) def test_check_for_preexist_dir_file(self): cmd_vrfy('mkdir -p test_data/dir') self.assertTrue( os.path.exists('test_data/dir') ) @@ -38,8 +52,8 @@ def test_filesys_cmds(self): dPATH=f'{self.PATH}/test_data/dir' mkdir_vrfy(dPATH) self.assertTrue( os.path.exists(dPATH) ) - cp_vrfy(f'{self.PATH}/change_case.py', f'{dPATH}/change_cases.py') - self.assertTrue( os.path.exists(f'{dPATH}/change_cases.py') ) + cp_vrfy(f'{self.PATH}/misc.py', f'{dPATH}/miscs.py') + self.assertTrue( os.path.exists(f'{dPATH}/miscs.py') ) cmd_vrfy(f'rm -rf {dPATH}') self.assertFalse( os.path.exists('tt.py') ) def test_get_charvar_from_netcdf(self): @@ -53,13 +67,6 @@ def test_get_elem_inds(self): self.assertEqual( get_elem_inds(arr, 'egg', 'first' ) , 0 ) self.assertEqual( get_elem_inds(arr, 'egg', 'last' ) , 4 ) self.assertEqual( get_elem_inds(arr, 'egg', 'all' ) , [0, 2, 4] ) - def test_get_manage_externals_config_property(self): - self.assertIn( \ - 'regional_workflow', - get_manage_externals_config_property( \ - f'{self.PATH}/test_data/Externals.cfg', - 'regional_workflow', - 'repo_url')) def test_interpol_to_arbit_CRES(self): RES = 800 RES_array = [ 5, 25, 40, 60, 80, 100, 400, 700, 1000, 1500, 2800, 3000 ] @@ -95,7 +102,7 @@ def test_import_vars(self): set_env_var("MYVAR","MYVAL") env_vars = ["PWD", "MYVAR"] import_vars(env_vars=env_vars) - self.assertEqual( PWD, os.getcwd() ) + self.assertEqual( os.path.realpath(PWD), os.path.realpath(os.getcwd()) ) self.assertEqual(MYVAR,"MYVAL") #test export MYVAR="MYNEWVAL" @@ -106,10 +113,26 @@ def test_import_vars(self): dictionary = { "Hello": "World!" } import_vars(dictionary=dictionary) self.assertEqual( Hello, "World!" ) + #test array + shell_str='("1" "2") \n' + v = str_to_list(shell_str) + self.assertTrue( isinstance(v,list) ) + self.assertEqual(v, [1, 2]) + shell_str = '( "1" "2" \n' + v = str_to_list(shell_str) + self.assertFalse( isinstance(v,list) ) def test_config_parser(self): cfg = { "HRS": [ "1", "2" ] } shell_str = cfg_to_shell_str(cfg) self.assertEqual( shell_str, 'HRS=( "1" "2" )\n') + # ini file + cfg = load_ini_config(f'{self.PATH}/test_data/Externals.cfg') + self.assertIn( \ + 'regional_workflow', + get_ini_value( \ + cfg, + 'regional_workflow', + 'repo_url')) def test_print_msg(self): self.assertEqual( print_info_msg("Hello World!", verbose=False), False) def setUp(self): diff --git a/ush/python_utils/xml_parser.py b/ush/python_utils/xml_parser.py new file mode 100644 index 0000000000..0428259c6d --- /dev/null +++ b/ush/python_utils/xml_parser.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import xml.etree.ElementTree as ET + +def load_xml_file(xml_file): + """ Loads xml file + + Args: + xml_file: path to xml file + Returns: + root of the xml tree + """ + tree = ET.parse(xml_file) + return tree + +def has_tag_with_value(tree, tag, value): + """ Check if xml tree has a node with tag and value + + Args: + tree: the xml tree + tag: the tag + value: text of tag + Returns: + Boolean + """ + for node in tree.iter(): + if node.tag == tag and node.text == value: + return True + return False + diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py new file mode 100644 index 0000000000..9d9ae4b39e --- /dev/null +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import unittest +import os +from textwrap import dedent +from datetime import datetime + +from python_utils import print_input_args, print_info_msg, print_err_msg_exit,\ + date_to_str, mkdir_vrfy,cp_vrfy,\ + import_vars,set_env_var,\ + define_macos_utilities, cfg_to_yaml_str + +from set_namelist import set_namelist + +def set_FV3nml_ens_stoch_seeds(cdate): + """ + This function, for an ensemble-enabled experiment + (i.e. for an experiment for which the workflow configuration variable + DO_ENSEMBLE has been set to "TRUE"), creates new namelist files with + unique stochastic "seed" parameters, using a base namelist file in the + ${EXPTDIR} directory as a template. These new namelist files are stored + within each member directory housed within each cycle directory. Files + of any two ensemble members differ only in their stochastic "seed" + parameter values. These namelist files are generated when this file is + called as part of the RUN_FCST_TN task. + + Args: + cdate + Returns: + None + """ + + print_input_args(locals()) + + # import all environment variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # For a given cycle and member, generate a namelist file with unique + # seed values. + # + #----------------------------------------------------------------------- + # + ensmem_name=f"mem{ENSMEM_INDX}" + + fv3_nml_ensmem_fp=os.path.join(CYCLE_BASEDIR, f"{date_to_str(cdate,True)}{os.sep}{ensmem_name}{os.sep}{FV3_NML_FN}") + + ensmem_num=ENSMEM_INDX + + cdate_i = int(cdate.strftime('%Y%m%d')) + + settings = {} + nam_stochy_dict = {} + + if DO_SPP: + iseed_sppt=cdate_i*1000 + ensmem_num*10 + 1 + nam_stochy_dict.update({ + 'iseed_sppt': iseed_sppt + }) + + if DO_SHUM: + iseed_shum=cdate_i*1000 + ensmem_num*10 + 2 + nam_stochy_dict.update({ + 'iseed_shum': iseed_shum + }) + + if DO_SKEB: + iseed_skeb=cdate_i*1000 + ensmem_num*10 + 3 + nam_stochy_dict.update({ + 'iseed_skeb': iseed_skeb + }) + + settings['nam_stochy'] = nam_stochy_dict + + if DO_SPP: + num_iseed_spp=len(ISEED_SPP) + iseed_spp = [None]*num_iseed_spp + for i in range(num_iseed_spp): + iseed_spp[i]=cdate_i*1000 + ensmem_num*10 + ISEED_SPP[i] + + settings['nam_spperts'] = { + 'iseed_spp': iseed_spp + } + else: + settings['nam_spperts'] = {} + + if DO_LSM_SPP: + iseed_lsm_spp=cdate_i*1000 + ensmem_num*10 + 9 + + settings['nam_sppperts'] = { + 'iseed_lndp': [iseed_lsm_spp] + } + + settings_str = cfg_to_yaml_str(settings) + try: + set_namelist(["-q", "-n", FV3_NML_FP, "-u", settings_str, "-o", fv3_nml_ensmem_fp]) + except: + print_err_msg_exit(dedent(f''' + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_FP = \"{FV3_NML_FP}\" + Full path to output namelist file: + fv3_nml_ensmem_fp = \"{fv3_nml_ensmem_fp}\" + Namelist settings specified on command line (these have highest precedence): + settings = + {settings_str}''')) + +class Testing(unittest.TestCase): + def test_set_FV3nml_ens_stoch_seeds(self): + set_FV3nml_ens_stoch_seeds(cdate=self.cdate) + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + USHDIR = os.path.dirname(os.path.abspath(__file__)) + EXPTDIR = os.path.join(USHDIR,"test_data","expt"); + cp_vrfy(os.path.join(USHDIR,f'templates{os.sep}input.nml.FV3'), \ + os.path.join(EXPTDIR,'input.nml')) + self.cdate=datetime(2021, 1, 1) + + mkdir_vrfy("-p", os.path.join(EXPTDIR,f'{date_to_str(self.cdate,True)}{os.sep}mem0')) + set_env_var("USHDIR",USHDIR) + set_env_var("CYCLE_BASEDIR",EXPTDIR) + set_env_var("ENSMEM_INDX",0) + set_env_var("FV3_NML_FN","input.nml") + set_env_var("FV3_NML_FP",os.path.join(EXPTDIR,"input.nml")) + set_env_var("DO_SPP",True) + set_env_var("DO_SHUM",True) + set_env_var("DO_SKEB",True) + set_env_var("DO_LSM_SPP",True) + ISEED_SPP = [ 4, 4, 4, 4, 4] + set_env_var("ISEED_SPP",ISEED_SPP) + diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py new file mode 100644 index 0000000000..518f81ae36 --- /dev/null +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +import unittest +import os +from textwrap import dedent + +from python_utils import print_input_args, print_info_msg, print_err_msg_exit,\ + check_var_valid_value,mv_vrfy,mkdir_vrfy,cp_vrfy,\ + rm_vrfy,import_vars,set_env_var,\ + define_macos_utilities,find_pattern_in_str,cfg_to_yaml_str + +from set_namelist import set_namelist + +def set_FV3nml_sfc_climo_filenames(): + """ + This function sets the values of the variables in + the forecast model's namelist file that specify the paths to the surface + climatology files on the FV3LAM native grid (which are either pregenerated + or created by the MAKE_SFC_CLIMO_TN task). Note that the workflow + generation scripts create symlinks to these surface climatology files + in the FIXLAM directory, and the values in the namelist file that get + set by this function are relative or full paths to these links. + + Args: + None + Returns: + None + """ + + # import all environment variables + import_vars() + + # The regular expression regex_search set below will be used to extract + # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING + # the name of the namelist variable to set and the corresponding surface + # climatology field from which to form the name of the surface climatology file + regex_search = "^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" + + # Set the suffix of the surface climatology files. + suffix = "tileX.nc" + + # create yaml-complaint string + settings = {} + + dummy_run_dir = os.path.join(EXPTDIR, "any_cyc") + if DO_ENSEMBLE == "TRUE": + dummy_run_dir += os.sep + "any_ensmem" + + namsfc_dict = {} + for mapping in FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: + tup = find_pattern_in_str(regex_search, mapping) + nml_var_name = tup[0] + sfc_climo_field_name = tup[1] + + check_var_valid_value(sfc_climo_field_name, SFC_CLIMO_FIELDS) + + fp = os.path.join(FIXLAM, f'{CRES}.{sfc_climo_field_name}.{suffix}') + if RUN_ENVIR != "nco": + fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) + + namsfc_dict[nml_var_name] = fp + + settings['namsfc_dict'] = namsfc_dict + settings_str = cfg_to_yaml_str(settings) + + + print_info_msg(f''' + The variable \"settings\" specifying values of the namelist variables + has been set as follows: + + settings = + {settings_str}''', verbose=DEBUG) + + # Rename the FV3 namelist and call set_namelist + fv3_nml_base_fp = f'{FV3_NML_FP}.base' + mv_vrfy(f'{FV3_NML_FP} {fv3_nml_base_fp}') + + try: + set_namelist(["-q", "-n", fv3_nml_base_fp, "-u", settings_str, "-o", FV3_NML_FP]) + except: + print_err_msg_exit(f''' + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + fv3_nml_base_fp = \"{fv3_nml_base_fp}\" + Full path to output namelist file: + FV3_NML_FP = \"{FV3_NML_FP}\" + Namelist settings specified on command line (these have highest precedence): + settings = + {settings_str}''') + + rm_vrfy(f'{fv3_nml_base_fp}') + +class Testing(unittest.TestCase): + def test_set_FV3nml_sfc_climo_filenames(self): + set_FV3nml_sfc_climo_filenames() + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + USHDIR = os.path.dirname(os.path.abspath(__file__)) + EXPTDIR = os.path.join(USHDIR, "test_data", "expt"); + FIXLAM = os.path.join(EXPTDIR, "fix_lam") + mkdir_vrfy("-p",FIXLAM) + cp_vrfy(os.path.join(USHDIR,f'templates{os.sep}input.nml.FV3'), \ + os.path.join(EXPTDIR,'input.nml')) + set_env_var("USHDIR",USHDIR) + set_env_var("EXPTDIR",EXPTDIR) + set_env_var("FIXLAM",FIXLAM) + set_env_var("DO_ENSEMBLE",False) + set_env_var("CRES","C3357") + set_env_var("RUN_ENVIR","nco") + set_env_var("FV3_NML_FP",os.path.join(EXPTDIR,"input.nml")) + + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING=[ + "FNALBC | snowfree_albedo", + "FNALBC2 | facsf", + "FNTG3C | substrate_temperature", + "FNVEGC | vegetation_greenness", + "FNVETC | vegetation_type", + "FNSOTC | soil_type", + "FNVMNC | vegetation_greenness", + "FNVMXC | vegetation_greenness", + "FNSLPC | slope_type", + "FNABSC | maximum_snow_albedo" + ] + SFC_CLIMO_FIELDS=[ + "facsf", + "maximum_snow_albedo", + "slope_type", + "snowfree_albedo", + "soil_type", + "substrate_temperature", + "vegetation_greenness", + "vegetation_type" + ] + set_env_var("FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING", + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING) + set_env_var("SFC_CLIMO_FIELDS",SFC_CLIMO_FIELDS) + diff --git a/ush/set_cycle_dates.py b/ush/set_cycle_dates.py new file mode 100644 index 0000000000..69a707b972 --- /dev/null +++ b/ush/set_cycle_dates.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import unittest +from datetime import datetime,timedelta,date + +from python_utils import print_input_args, print_err_msg_exit + +def set_cycle_dates(date_start, date_end, cycle_hrs, incr_cycl_freq): + """ This file defines a function that, given the starting date (date_start, + in the form YYYYMMDD), the ending date (date_end, in the form YYYYMMDD), + and an array containing the cycle hours for each day (whose elements + have the form HH), returns an array of cycle date-hours whose elements + have the form YYYYMMDD. Here, YYYY is a four-digit year, MM is a two- + digit month, DD is a two-digit day of the month, and HH is a two-digit + hour of the day. + + Args: + date_start: start date + date_end: end date + cycle_hrs: [ HH0, HH1, ...] + incr_cycl_freq: cycle frequency increment in hours + Returns: + A list of dates in a format YYYYMMDDHH + """ + + print_input_args(locals()) + + #calculate date increment + if incr_cycl_freq <= 24: + incr_days = 1 + else: + incr_days = incr_cycl_freq // 24 + if incr_cycl_freq % 24 != 0: + print_err_msg_exit(f''' + INCR_CYCL_FREQ is not divided by 24: + INCR_CYCL_FREQ = \"{incr_cycl_freq}\"''') + + #iterate over days and cycles + all_cdates = [] + d = date_start + while d <= date_end: + for c in cycle_hrs: + dc = d + timedelta(hours=c) + v = datetime.strftime(dc,'%Y%m%d%H') + all_cdates.append(v) + d += timedelta(days=incr_days) + + return all_cdates + +class Testing(unittest.TestCase): + def test_set_cycle_dates(self): + cdates = set_cycle_dates(date_start=datetime(2022,1,1), date_end=datetime(2022,1,4), + cycle_hrs=[6,12], incr_cycl_freq=48) + self.assertEqual(cdates, ['2022010106', '2022010112','2022010306', '2022010312']) diff --git a/ush/set_extrn_mdl_params.py b/ush/set_extrn_mdl_params.py new file mode 100644 index 0000000000..1eea1f2bab --- /dev/null +++ b/ush/set_extrn_mdl_params.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +import unittest + +from python_utils import import_vars, export_vars, set_env_var, get_env_var + +def set_extrn_mdl_params(): + """ Sets parameters associated with the external model used for initial + conditions (ICs) and lateral boundary conditions (LBCs). + Args: + None + Returns: + None + """ + + #import all env variables + import_vars() + + global EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS, EXTRN_MDL_LBCS_OFFSET_HRS + + # + #----------------------------------------------------------------------- + # + # Use known locations or COMIN as default, depending on RUN_ENVIR + # + #----------------------------------------------------------------------- + # + if RUN_ENVIR == "nco": + EXTRN_MDL_SYSBASEDIR_ICS=EXTRN_MDL_SYSBASEDIR_ICS or COMIN + EXTRN_MDL_SYSBASEDIR_LBCS=EXTRN_MDL_SYSBASEDIR_LBCS or COMIN + else: + ics_dir=EXTRN_MDL_NAME_ICS + lbcs_dir=EXTRN_MDL_NAME_LBCS + EXTRN_MDL_SYSBASEDIR_ICS=EXTRN_MDL_SYSBASEDIR_ICS or ics_dir + EXTRN_MDL_SYSBASEDIR_LBCS=EXTRN_MDL_SYSBASEDIR_LBCS or lbcs_dir + + # + #----------------------------------------------------------------------- + # + # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift + # the starting time of the external model that provides lateral boundary + # conditions. + # + #----------------------------------------------------------------------- + # + if EXTRN_MDL_NAME_LBCS == "RAP": + EXTRN_MDL_LBCS_OFFSET_HRS=EXTRN_MDL_LBCS_OFFSET_HRS or "3" + else: + EXTRN_MDL_LBCS_OFFSET_HRS=EXTRN_MDL_LBCS_OFFSET_HRS or "0" + + # export values we set above + env_vars = ["EXTRN_MDL_SYSBASEDIR_ICS", "EXTRN_MDL_SYSBASEDIR_LBCS", "EXTRN_MDL_LBCS_OFFSET_HRS"] + export_vars(env_vars=env_vars) +# +#----------------------------------------------------------------------- +# +# Call the function defined above. +# +#----------------------------------------------------------------------- +# +if __name__ == "__main__": + set_extrn_mdl_params() + +class Testing(unittest.TestCase): + def test_extrn_mdl_params(self): + set_extrn_mdl_params() + EXTRN_MDL_SYSBASEDIR_ICS = get_env_var("EXTRN_MDL_SYSBASEDIR_ICS") + COMINgfs = get_env_var("COMINgfs") + self.assertEqual(EXTRN_MDL_SYSBASEDIR_ICS,COMINgfs) + + def setUp(self): + set_env_var("MACHINE","HERA") + set_env_var("RUN_ENVIR","nco") + set_env_var("EXTRN_MDL_NAME_ICS","FV3GFS") + set_env_var("EXTRN_MDL_NAME_LBCS","FV3GFS") + set_env_var("EXTRN_MDL_SYSBASEDIR_ICS",None) + set_env_var("EXTRN_MDL_SYSBASEDIR_LBCS",None) + set_env_var("EXTRN_MDL_LBCS_OFFSET_HRS",None) + set_env_var("COMINgfs","/base/path/of/directory/containing/gfs/input/files") diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 9664d6a471..725759dc99 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -36,7 +36,7 @@ function set_extrn_mdl_params() { "RAP") EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"3"} ;; - "*") + *) EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"0"} ;; esac diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py new file mode 100644 index 0000000000..c1ead9522f --- /dev/null +++ b/ush/set_gridparams_ESGgrid.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import unittest +from datetime import datetime,timedelta + +from constants import radius_Earth,degs_per_radian +from python_utils import import_vars, set_env_var, print_input_args + +def set_gridparams_ESGgrid(lon_ctr,lat_ctr,nx,ny,halo_width,delx,dely,pazi): + """ Sets the parameters for a grid that is to be generated using the "ESGgrid" + grid generation method (i.e. GRID_GEN_METHOD set to "ESGgrid"). + + Args: + lon_ctr + lat_ctr + nx + ny + halo_width + delx + dely + pazi + Returns: + Tuple of inputs, and 4 outputs (see return statement) + """ + + print_input_args(locals()) + # + #----------------------------------------------------------------------- + # + # For a ESGgrid-type grid, the orography filtering is performed by pass- + # ing to the orography filtering the parameters for an "equivalent" glo- + # bal uniform cubed-sphere grid. These are the parameters that a global + # uniform cubed-sphere grid needs to have in order to have a nominal + # grid cell size equal to that of the (average) cell size on the region- + # al grid. These globally-equivalent parameters include a resolution + # (in units of number of cells in each of the two horizontal directions) + # and a stretch factor. The equivalent resolution is calculated in the + # script that generates the grid, and the stretch factor needs to be set + # to 1 because we are considering an equivalent globally UNIFORM grid. + # However, it turns out that with a non-symmetric regional grid (one in + # which nx is not equal to ny), setting stretch_factor to 1 fails be- + # cause the orography filtering program is designed for a global cubed- + # sphere grid and thus assumes that nx and ny for a given tile are equal + # when stretch_factor is exactly equal to 1. + # ^^-- Why is this? Seems like symmetry btwn x and y should still hold when the stretch factor is not equal to 1. + # It turns out that the program will work if we set stretch_factor to a + # value that is not exactly 1. This is what we do below. + # + #----------------------------------------------------------------------- + # + stretch_factor=0.999 # Check whether the orography program has been fixed so that we can set this to 1... + # + #----------------------------------------------------------------------- + # + # Set parameters needed as inputs to the regional_grid grid generation + # code. + # + #----------------------------------------------------------------------- + # + del_angle_x_sg = (delx / (2.0 * radius_Earth)) * degs_per_radian + del_angle_y_sg = (dely / (2.0 * radius_Earth)) * degs_per_radian + neg_nx_of_dom_with_wide_halo = -(nx + 2 * halo_width) + neg_ny_of_dom_with_wide_halo = -(ny + 2 * halo_width) + # + #----------------------------------------------------------------------- + # + # return output variables. + # + #----------------------------------------------------------------------- + # + return (lon_ctr,lat_ctr,nx,ny,pazi,halo_width,stretch_factor, + "{:0.10f}".format(del_angle_x_sg), + "{:0.10f}".format(del_angle_y_sg), + "{:.0f}".format(neg_nx_of_dom_with_wide_halo), + "{:.0f}".format(neg_ny_of_dom_with_wide_halo)) + +class Testing(unittest.TestCase): + def test_set_gridparams_ESGgrid(self): + + (LON_CTR,LAT_CTR,NX,NY,PAZI,NHW,STRETCH_FAC, + DEL_ANGLE_X_SG, + DEL_ANGLE_Y_SG, + NEG_NX_OF_DOM_WITH_WIDE_HALO, + NEG_NY_OF_DOM_WITH_WIDE_HALO) = set_gridparams_ESGgrid( \ + lon_ctr=-97.5, + lat_ctr=38.5, + nx=1748, + ny=1038, + pazi=0.0, + halo_width=6, + delx=3000.0, + dely=3000.0) + + self.assertEqual(\ + (LON_CTR,LAT_CTR,NX,NY,PAZI,NHW,STRETCH_FAC, + DEL_ANGLE_X_SG, + DEL_ANGLE_Y_SG, + NEG_NX_OF_DOM_WITH_WIDE_HALO, + NEG_NY_OF_DOM_WITH_WIDE_HALO), + (-97.5, 38.5, 1748, 1038, 0.0, 6,0.999, + "0.0134894006", + "0.0134894006", + "-1760", + "-1050") + ) + + def setUp(self): + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py new file mode 100644 index 0000000000..0dab11f756 --- /dev/null +++ b/ush/set_gridparams_GFDLgrid.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 + +import unittest + +from constants import radius_Earth,degs_per_radian + +from python_utils import import_vars, set_env_var, print_input_args, \ + print_info_msg, print_err_msg_exit + +def prime_factors(n): + i = 2 + factors = [] + while i * i <= n: + if n % i: + i += 1 + else: + n //= i + factors.append(i) + if n > 1: + factors.append(n) + return factors + +def set_gridparams_GFDLgrid(lon_of_t6_ctr, lat_of_t6_ctr, res_of_t6g, stretch_factor, + refine_ratio_t6g_to_t7g, + istart_of_t7_on_t6g, iend_of_t7_on_t6g, + jstart_of_t7_on_t6g, jend_of_t7_on_t6g): + """ Sets the parameters for a grid that is to be generated using the "GFDLgrid" + grid generation method (i.e. GRID_GEN_METHOD set to "ESGgrid"). + + Args: + lon_of_t6_ctr + lat_of_t6_ctr + res_of_t6g + stretch_factor + refine_ratio_t6g_to_t7g + istart_of_t7_on_t6g + iend_of_t7_on_t6g + jstart_of_t7_on_t6g + jend_of_t7_on_t6g): + Returns: + Tuple of inputs and outputs (see return statement) + """ + + print_input_args(locals()) + + # get needed environment variables + IMPORTS = ['VERBOSE', 'RUN_ENVIR', 'NH4'] + import_vars(env_vars=IMPORTS) + + # + #----------------------------------------------------------------------- + # + # To simplify the grid setup, we require that tile 7 be centered on tile + # 6. Note that this is not really a restriction because tile 6 can al- + # ways be moved so that it is centered on tile 7 [the location of tile 6 + # doesn't really matter because for a regional setup, the forecast model + # will only run on tile 7 (not on tiles 1-6)]. + # + # We now check that tile 7 is centered on tile 6 by checking (1) that + # the number of cells (on tile 6) between the left boundaries of these + # two tiles is equal to that between their right boundaries and (2) that + # the number of cells (on tile 6) between the bottom boundaries of these + # two tiles is equal to that between their top boundaries. If not, we + # print out an error message and exit. If so, we set the longitude and + # latitude of the center of tile 7 to those of tile 6 and continue. + # + #----------------------------------------------------------------------- + # + + nx_of_t6_on_t6g = res_of_t6g + ny_of_t6_on_t6g = res_of_t6g + + num_left_margin_cells_on_t6g = istart_of_t7_on_t6g - 1 + num_right_margin_cells_on_t6g = nx_of_t6_on_t6g - iend_of_t7_on_t6g + + # This if-statement can hopefully be removed once EMC agrees to make their + # GFDLgrid type grids (tile 7) symmetric about tile 6. + if RUN_ENVIR != "nco": + if num_left_margin_cells_on_t6g != num_right_margin_cells_on_t6g: + print_err_msg_exit(f''' + In order for tile 7 to be centered in the x direction on tile 6, the x- + direction tile 6 cell indices at which tile 7 starts and ends (given by + istart_of_t7_on_t6g and iend_of_t7_on_t6g, respectively) must be set + such that the number of tile 6 cells in the margin between the left + boundaries of tiles 6 and 7 (given by num_left_margin_cells_on_t6g) is + equal to that in the margin between their right boundaries (given by + num_right_margin_cells_on_t6g): + istart_of_t7_on_t6g = {istart_of_t7_on_t6g} + iend_of_t7_on_t6g = {iend_of_t7_on_t6g} + num_left_margin_cells_on_t6g = {num_left_margin_cells_on_t6g} + num_right_margin_cells_on_t6g = {num_right_margin_cells_on_t6g} + Note that the total number of cells in the x-direction on tile 6 is gi- + ven by: + nx_of_t6_on_t6g = {nx_of_t6_on_t6g} + Please reset istart_of_t7_on_t6g and iend_of_t7_on_t6g and rerun.''') + + num_bot_margin_cells_on_t6g = jstart_of_t7_on_t6g - 1 + num_top_margin_cells_on_t6g = ny_of_t6_on_t6g - jend_of_t7_on_t6g + + # This if-statement can hopefully be removed once EMC agrees to make their + # GFDLgrid type grids (tile 7) symmetric about tile 6. + if RUN_ENVIR != "nco": + if num_bot_margin_cells_on_t6g != num_top_margin_cells_on_t6g: + print_err_msg_exit(f''' + In order for tile 7 to be centered in the y direction on tile 6, the y- + direction tile 6 cell indices at which tile 7 starts and ends (given by + jstart_of_t7_on_t6g and jend_of_t7_on_t6g, respectively) must be set + such that the number of tile 6 cells in the margin between the left + boundaries of tiles 6 and 7 (given by num_left_margin_cells_on_t6g) is + equal to that in the margin between their right boundaries (given by + num_right_margin_cells_on_t6g): + jstart_of_t7_on_t6g = {jstart_of_t7_on_t6g} + jend_of_t7_on_t6g = {jend_of_t7_on_t6g} + num_bot_margin_cells_on_t6g = {num_bot_margin_cells_on_t6g} + num_top_margin_cells_on_t6g = {num_top_margin_cells_on_t6g} + Note that the total number of cells in the y-direction on tile 6 is gi- + ven by: + ny_of_t6_on_t6g = {ny_of_t6_on_t6g} + Please reset jstart_of_t7_on_t6g and jend_of_t7_on_t6g and rerun.''') + + lon_of_t7_ctr = lon_of_t6_ctr + lat_of_t7_ctr = lat_of_t6_ctr + # + #----------------------------------------------------------------------- + # + # The grid generation script grid_gen_scr called below in turn calls the + # make_hgrid utility/executable to construct the regional grid. make_- + # hgrid accepts as arguments the index limits (i.e. starting and ending + # indices) of the regional grid on the supergrid of the regional grid's + # parent tile. The regional grid's parent tile is tile 6, and the su- + # pergrid of any given tile is defined as the grid obtained by doubling + # the number of cells in each direction on that tile's grid. We will + # denote these index limits by + # + # istart_of_t7_on_t6sg + # iend_of_t7_on_t6sg + # jstart_of_t7_on_t6sg + # jend_of_t7_on_t6sg + # + # The "_T6SG" suffix in these names is used to indicate that the indices + # are on the supergrid of tile 6. Recall, however, that we have as in- + # puts the index limits of the regional grid on the tile 6 grid, not its + # supergrid. These are given by + # + # istart_of_t7_on_t6g + # iend_of_t7_on_t6g + # jstart_of_t7_on_t6g + # jend_of_t7_on_t6g + # + # We can obtain the former from the latter by recalling that the super- + # grid has twice the resolution of the original grid. Thus, + # + # istart_of_t7_on_t6sg = 2*istart_of_t7_on_t6g - 1 + # iend_of_t7_on_t6sg = 2*iend_of_t7_on_t6g + # jstart_of_t7_on_t6sg = 2*jstart_of_t7_on_t6g - 1 + # jend_of_t7_on_t6sg = 2*jend_of_t7_on_t6g + # + # These are obtained assuming that grid cells on tile 6 must either be + # completely within the regional domain or completely outside of it, + # i.e. the boundary of the regional grid must coincide with gridlines + # on the tile 6 grid; it cannot cut through tile 6 cells. (Note that + # this implies that the starting indices on the tile 6 supergrid must be + # odd while the ending indices must be even; the above expressions sa- + # tisfy this requirement.) We perfrom these calculations next. + # + #----------------------------------------------------------------------- + # + istart_of_t7_on_t6sg = 2*istart_of_t7_on_t6g - 1 + iend_of_t7_on_t6sg = 2*iend_of_t7_on_t6g + jstart_of_t7_on_t6sg = 2*jstart_of_t7_on_t6g - 1 + jend_of_t7_on_t6sg = 2*jend_of_t7_on_t6g + # + #----------------------------------------------------------------------- + # + # If we simply pass to make_hgrid the index limits of the regional grid + # on the tile 6 supergrid calculated above, make_hgrid will generate a + # regional grid without a halo. To obtain a regional grid with a halo, + # we must pass to make_hgrid the index limits (on the tile 6 supergrid) + # of the regional grid including a halo. We will let the variables + # + # istart_of_t7_with_halo_on_t6sg + # iend_of_t7_with_halo_on_t6sg + # jstart_of_t7_with_halo_on_t6sg + # jend_of_t7_with_halo_on_t6sg + # + # denote these limits. The reason we include "_wide_halo" in these va- + # riable names is that the halo of the grid that we will first generate + # will be wider than the halos that are actually needed as inputs to the + # FV3LAM model (i.e. the 0-cell-wide, 3-cell-wide, and 4-cell-wide halos + # described above). We will generate the grids with narrower halos that + # the model needs later on by "shaving" layers of cells from this wide- + # halo grid. Next, we describe how to calculate the above indices. + # + # Let halo_width_on_t7g denote the width of the "wide" halo in units of number of + # grid cells on the regional grid (i.e. tile 7) that we'd like to have + # along all four edges of the regional domain (left, right, bottom, and + # top). To obtain the corresponding halo width in units of number of + # cells on the tile 6 grid -- which we denote by halo_width_on_t6g -- we simply di- + # vide halo_width_on_t7g by the refinement ratio, i.e. + # + # halo_width_on_t6g = halo_width_on_t7g/refine_ratio_t6g_to_t7g + # + # The corresponding halo width on the tile 6 supergrid is then given by + # + # halo_width_on_t6sg = 2*halo_width_on_t6g + # = 2*halo_width_on_t7g/refine_ratio_t6g_to_t7g + # + # Note that halo_width_on_t6sg must be an integer, but the expression for it de- + # rived above may not yield an integer. To ensure that the halo has a + # width of at least halo_width_on_t7g cells on the regional grid, we round up the + # result of the expression above for halo_width_on_t6sg, i.e. we redefine halo_width_on_t6sg + # to be + # + # halo_width_on_t6sg = ceil(2*halo_width_on_t7g/refine_ratio_t6g_to_t7g) + # + # where ceil(...) is the ceiling function, i.e. it rounds its floating + # point argument up to the next larger integer. Since in bash division + # of two integers returns a truncated integer and since bash has no + # built-in ceil(...) function, we perform the rounding-up operation by + # adding the denominator (of the argument of ceil(...) above) minus 1 to + # the original numerator, i.e. by redefining halo_width_on_t6sg to be + # + # halo_width_on_t6sg = (2*halo_width_on_t7g + refine_ratio_t6g_to_t7g - 1)/refine_ratio_t6g_to_t7g + # + # This trick works when dividing one positive integer by another. + # + # In order to calculate halo_width_on_t6g using the above expression, we must + # first specify halo_width_on_t7g. Next, we specify an initial value for it by + # setting it to one more than the largest-width halo that the model ac- + # tually needs, which is NH4. We then calculate halo_width_on_t6sg using the + # above expression. Note that these values of halo_width_on_t7g and halo_width_on_t6sg will + # likely not be their final values; their final values will be calcula- + # ted later below after calculating the starting and ending indices of + # the regional grid with wide halo on the tile 6 supergrid and then ad- + # justing the latter to satisfy certain conditions. + # + #----------------------------------------------------------------------- + # + halo_width_on_t7g = NH4 + 1 + halo_width_on_t6sg = (2*halo_width_on_t7g + refine_ratio_t6g_to_t7g - 1)/refine_ratio_t6g_to_t7g + # + #----------------------------------------------------------------------- + # + # With an initial value of halo_width_on_t6sg now available, we can obtain the + # tile 6 supergrid index limits of the regional domain (including the + # wide halo) from the index limits for the regional domain without a ha- + # lo by simply subtracting halo_width_on_t6sg from the lower index limits and add- + # ing halo_width_on_t6sg to the upper index limits, i.e. + # + # istart_of_t7_with_halo_on_t6sg = istart_of_t7_on_t6sg - halo_width_on_t6sg + # iend_of_t7_with_halo_on_t6sg = iend_of_t7_on_t6sg + halo_width_on_t6sg + # jstart_of_t7_with_halo_on_t6sg = jstart_of_t7_on_t6sg - halo_width_on_t6sg + # jend_of_t7_with_halo_on_t6sg = jend_of_t7_on_t6sg + halo_width_on_t6sg + # + # We calculate these next. + # + #----------------------------------------------------------------------- + # + istart_of_t7_with_halo_on_t6sg = int(istart_of_t7_on_t6sg - halo_width_on_t6sg) + iend_of_t7_with_halo_on_t6sg = int(iend_of_t7_on_t6sg + halo_width_on_t6sg) + jstart_of_t7_with_halo_on_t6sg = int(jstart_of_t7_on_t6sg - halo_width_on_t6sg) + jend_of_t7_with_halo_on_t6sg = int(jend_of_t7_on_t6sg + halo_width_on_t6sg) + # + #----------------------------------------------------------------------- + # + # As for the regional grid without a halo, the regional grid with a wide + # halo that make_hgrid will generate must be such that grid cells on + # tile 6 either lie completely within this grid or outside of it, i.e. + # they cannot lie partially within/outside of it. This implies that the + # starting indices on the tile 6 supergrid of the grid with wide halo + # must be odd while the ending indices must be even. Thus, below, we + # subtract 1 from the starting indices if they are even (which ensures + # that there will be at least halo_width_on_t7g halo cells along the left and bot- + # tom boundaries), and we add 1 to the ending indices if they are odd + # (which ensures that there will be at least halo_width_on_t7g halo cells along the + # right and top boundaries). + # + #----------------------------------------------------------------------- + # + if istart_of_t7_with_halo_on_t6sg % 2 == 0: + istart_of_t7_with_halo_on_t6sg = istart_of_t7_with_halo_on_t6sg - 1 + + if iend_of_t7_with_halo_on_t6sg % 2 == 1: + iend_of_t7_with_halo_on_t6sg = iend_of_t7_with_halo_on_t6sg + 1 + + if jstart_of_t7_with_halo_on_t6sg % 2 == 0: + jstart_of_t7_with_halo_on_t6sg = jstart_of_t7_with_halo_on_t6sg - 1 + + if jend_of_t7_with_halo_on_t6sg % 2 == 1: + jend_of_t7_with_halo_on_t6sg = jend_of_t7_with_halo_on_t6sg + 1 + # + #----------------------------------------------------------------------- + # + # Now that the starting and ending tile 6 supergrid indices of the re- + # gional grid with the wide halo have been calculated (and adjusted), we + # recalculate the width of the wide halo on: + # + # 1) the tile 6 supergrid; + # 2) the tile 6 grid; and + # 3) the tile 7 grid. + # + # These are the final values of these quantities that are guaranteed to + # correspond to the starting and ending indices on the tile 6 supergrid. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + Original values of the halo width on the tile 6 supergrid and on the + tile 7 grid are: + halo_width_on_t6sg = {halo_width_on_t6sg} + halo_width_on_t7g = {halo_width_on_t7g}''', verbose=VERBOSE) + + halo_width_on_t6sg = istart_of_t7_on_t6sg - istart_of_t7_with_halo_on_t6sg + halo_width_on_t6g = halo_width_on_t6sg//2 + halo_width_on_t7g = int(halo_width_on_t6g*refine_ratio_t6g_to_t7g) + + print_info_msg(f''' + Values of the halo width on the tile 6 supergrid and on the tile 7 grid + AFTER adjustments are: + halo_width_on_t6sg = {halo_width_on_t6sg} + halo_width_on_t7g = {halo_width_on_t7g}''', verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # Calculate the number of cells that the regional domain (without halo) + # has in each of the two horizontal directions (say x and y). We denote + # these by nx_of_t7_on_t7g and ny_of_t7_on_t7g, respectively. These + # will be needed in the "shave" steps in the grid generation task of the + # workflow. + # + #----------------------------------------------------------------------- + # + nx_of_t7_on_t6sg = iend_of_t7_on_t6sg - istart_of_t7_on_t6sg + 1 + nx_of_t7_on_t6g = nx_of_t7_on_t6sg/2 + nx_of_t7_on_t7g = int(nx_of_t7_on_t6g*refine_ratio_t6g_to_t7g) + + ny_of_t7_on_t6sg = jend_of_t7_on_t6sg - jstart_of_t7_on_t6sg + 1 + ny_of_t7_on_t6g = ny_of_t7_on_t6sg/2 + ny_of_t7_on_t7g = int(ny_of_t7_on_t6g*refine_ratio_t6g_to_t7g) + # + # The following are set only for informational purposes. + # + nx_of_t6_on_t6sg = 2*nx_of_t6_on_t6g + ny_of_t6_on_t6sg = 2*ny_of_t6_on_t6g + + prime_factors_nx_of_t7_on_t7g = prime_factors(nx_of_t7_on_t7g) + prime_factors_ny_of_t7_on_t7g = prime_factors(ny_of_t7_on_t7g) + + print_info_msg(f''' + The number of cells in the two horizontal directions (x and y) on the + parent tile's (tile 6) grid and supergrid are: + nx_of_t6_on_t6g = {nx_of_t6_on_t6g} + ny_of_t6_on_t6g = {ny_of_t6_on_t6g} + nx_of_t6_on_t6sg = {nx_of_t6_on_t6sg} + ny_of_t6_on_t6sg = {ny_of_t6_on_t6sg} + + The number of cells in the two horizontal directions on the tile 6 grid + and supergrid that the regional domain (tile 7) WITHOUT A HALO encompas- + ses are: + nx_of_t7_on_t6g = {nx_of_t7_on_t6g} + ny_of_t7_on_t6g = {ny_of_t7_on_t6g} + nx_of_t7_on_t6sg = {nx_of_t7_on_t6sg} + ny_of_t7_on_t6sg = {ny_of_t7_on_t6sg} + + The starting and ending i and j indices on the tile 6 grid used to gene- + rate this regional grid are: + istart_of_t7_on_t6g = {istart_of_t7_on_t6g} + iend_of_t7_on_t6g = {iend_of_t7_on_t6g} + jstart_of_t7_on_t6g = {jstart_of_t7_on_t6g} + jend_of_t7_on_t6g = {jend_of_t7_on_t6g} + + The corresponding starting and ending i and j indices on the tile 6 su- + pergrid are: + istart_of_t7_on_t6sg = {istart_of_t7_on_t6sg} + iend_of_t7_on_t6sg = {iend_of_t7_on_t6sg} + jstart_of_t7_on_t6sg = {jstart_of_t7_on_t6sg} + jend_of_t7_on_t6sg = {jend_of_t7_on_t6sg} + + The refinement ratio (ratio of the number of cells in tile 7 that abut + a single cell in tile 6) is: + refine_ratio_t6g_to_t7g = {refine_ratio_t6g_to_t7g} + + The number of cells in the two horizontal directions on the regional do- + main's (i.e. tile 7's) grid WITHOUT A HALO are: + nx_of_t7_on_t7g = {nx_of_t7_on_t7g} + ny_of_t7_on_t7g = {ny_of_t7_on_t7g} + + The prime factors of nx_of_t7_on_t7g and ny_of_t7_on_t7g are (useful for + determining an MPI task layout): + prime_factors_nx_of_t7_on_t7g: {prime_factors_nx_of_t7_on_t7g} + prime_factors_ny_of_t7_on_t7g: {prime_factors_ny_of_t7_on_t7g}''', verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # For informational purposes, calculate the number of cells in each di- + # rection on the regional grid including the wide halo (of width halo_- + # width_on_t7g cells). We denote these by nx_of_t7_with_halo_on_t7g and + # ny_of_t7_with_halo_on_t7g, respectively. + # + #----------------------------------------------------------------------- + # + nx_of_t7_with_halo_on_t6sg = iend_of_t7_with_halo_on_t6sg - istart_of_t7_with_halo_on_t6sg + 1 + nx_of_t7_with_halo_on_t6g = nx_of_t7_with_halo_on_t6sg/2 + nx_of_t7_with_halo_on_t7g = nx_of_t7_with_halo_on_t6g*refine_ratio_t6g_to_t7g + + ny_of_t7_with_halo_on_t6sg = jend_of_t7_with_halo_on_t6sg - jstart_of_t7_with_halo_on_t6sg + 1 + ny_of_t7_with_halo_on_t6g = ny_of_t7_with_halo_on_t6sg/2 + ny_of_t7_with_halo_on_t7g = ny_of_t7_with_halo_on_t6g*refine_ratio_t6g_to_t7g + + print_info_msg(f''' + nx_of_t7_with_halo_on_t7g = {nx_of_t7_with_halo_on_t7g} + (istart_of_t7_with_halo_on_t6sg = {istart_of_t7_with_halo_on_t6sg}, + iend_of_t7_with_halo_on_t6sg = {iend_of_t7_with_halo_on_t6sg})''', verbose=VERBOSE) + + print_info_msg(f''' + ny_of_t7_with_halo_on_t7g = {ny_of_t7_with_halo_on_t7g} + (jstart_of_t7_with_halo_on_t6sg = {jstart_of_t7_with_halo_on_t6sg}, + jend_of_t7_with_halo_on_t6sg = {jend_of_t7_with_halo_on_t6sg})''', verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # Return output variables. + # + #----------------------------------------------------------------------- + # + return (lon_of_t7_ctr, lat_of_t7_ctr, nx_of_t7_on_t7g, ny_of_t7_on_t7g, + halo_width_on_t7g, stretch_factor, + istart_of_t7_with_halo_on_t6sg, + iend_of_t7_with_halo_on_t6sg, + jstart_of_t7_with_halo_on_t6sg, + jend_of_t7_with_halo_on_t6sg) + +class Testing(unittest.TestCase): + def test_set_gridparams_GFDLgrid(self): + (LON_CTR,LAT_CTR,NX,NY,NHW,STRETCH_FAC, + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG) = set_gridparams_GFDLgrid( \ + lon_of_t6_ctr=-97.5, \ + lat_of_t6_ctr=38.5, \ + res_of_t6g=96, \ + stretch_factor=1.4, \ + refine_ratio_t6g_to_t7g=3, \ + istart_of_t7_on_t6g=13, \ + iend_of_t7_on_t6g=84, \ + jstart_of_t7_on_t6g=17, \ + jend_of_t7_on_t6g=80) + + self.assertEqual(\ + (LON_CTR,LAT_CTR,NX,NY,NHW,STRETCH_FAC, + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG), + (-97.5,38.5,216,192,6,1.4, + 21, + 172, + 29, + 164) + ) + + def setUp(self): + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + set_env_var('NH4', 4) + diff --git a/ush/set_namelist.py b/ush/set_namelist.py index 43bbeb8a66..84c61975ec 100755 --- a/ush/set_namelist.py +++ b/ush/set_namelist.py @@ -69,6 +69,7 @@ import argparse import collections import os +import sys import f90nml import yaml @@ -137,7 +138,7 @@ def path_ok(arg): msg = f'{arg} is not a writable path!' raise argparse.ArgumentTypeError(msg) -def parse_args(): +def parse_args(argv): ''' Function maintains the arguments accepted by this script. Please see @@ -192,7 +193,7 @@ def parse_args(): action='store_true', help='If provided, suppress all output.', ) - return parser.parse_args() + return parser.parse_args(argv) def dict_diff(dict1, dict2): @@ -284,10 +285,15 @@ def update_dict(dest, newdict, quiet=False): dest[sect] = {} dest[sect][key] = value -def main(cla): +def set_namelist(argv): ''' Using input command line arguments (cla), update a Fortran namelist file. ''' + # parse argumetns + cla = parse_args(argv) + if cla.config: + cla.config, _ = config_exists(cla.config) + # Load base namelist into dict nml = f90nml.Namelist() if cla.nml is not None: @@ -324,7 +330,4 @@ def main(cla): if __name__ == '__main__': - cla = parse_args() - if cla.config: - cla.config, _ = config_exists(cla.config) - main(cla) + set_namelist(sys.argv[1:]) diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py new file mode 100644 index 0000000000..4c9998c3ca --- /dev/null +++ b/ush/set_ozone_param.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 + +import os +import unittest +from textwrap import dedent + +from python_utils import import_vars,export_vars,set_env_var,list_to_str,\ + print_input_args, print_info_msg, print_err_msg_exit,\ + define_macos_utilities,load_xml_file,has_tag_with_value,find_pattern_in_str + +def set_ozone_param(ccpp_phys_suite_fp): + """ Function that does the following: + (1) Determines the ozone parameterization being used by checking in the + CCPP physics suite XML. + + (2) Sets the name of the global ozone production/loss file in the FIXgsm + FIXgsm system directory to copy to the experiment's FIXam directory. + + (3) Resets the last element of the workflow array variable + FIXgsm_FILES_TO_COPY_TO_FIXam that contains the files to copy from + FIXgsm to FIXam (this last element is initially set to a dummy + value) to the name of the ozone production/loss file set in the + previous step. + + (4) Resets the element of the workflow array variable + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING (this array contains the + mapping between the symlinks to create in any cycle directory and + the files in the FIXam directory that are their targets) that + specifies the mapping for the ozone symlink/file such that the + target FIXam file name is set to the name of the ozone production/ + loss file set above. + + Args: + ccpp_phys_suite_fp: full path to CCPP physics suite + Returns: + ozone_param: a string + """ + + print_input_args(locals()) + + # import all environment variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # Get the name of the ozone parameterization being used. There are two + # possible ozone parameterizations: + # + # (1) A parameterization developed/published in 2015. Here, we refer to + # this as the 2015 parameterization. If this is being used, then we + # set the variable ozone_param to the string "ozphys_2015". + # + # (2) A parameterization developed/published sometime after 2015. Here, + # we refer to this as the after-2015 parameterization. If this is + # being used, then we set the variable ozone_param to the string + # "ozphys". + # + # We check the CCPP physics suite definition file (SDF) to determine the + # parameterization being used. If this file contains the line + # + # ozphys_2015 + # + # then the 2015 parameterization is being used. If it instead contains + # the line + # + # ozphys + # + # then the after-2015 parameterization is being used. (The SDF should + # contain exactly one of these lines; not both nor neither; we check for + # this.) + # + #----------------------------------------------------------------------- + # + tree = load_xml_file(ccpp_phys_suite_fp) + ozone_param = "" + if has_tag_with_value(tree, "scheme", "ozphys_2015"): + fixgsm_ozone_fn="ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77" + ozone_param = "ozphys_2015" + elif has_tag_with_value(tree, "scheme", "ozphys"): + fixgsm_ozone_fn="global_o3prdlos.f77" + ozone_param = "ozphys" + else: + print_err_msg_exit(f''' + Unknown or no ozone parameterization + specified in the CCPP physics suite file (ccpp_phys_suite_fp): + ccpp_phys_suite_fp = \"{ccpp_phys_suite_fp}\" + ozone_param = \"{ozone_param}\"''') + # + #----------------------------------------------------------------------- + # + # Set the last element of the array FIXgsm_FILES_TO_COPY_TO_FIXam to the + # name of the ozone production/loss file to copy from the FIXgsm to the + # FIXam directory. + # + #----------------------------------------------------------------------- + # + i=len(FIXgsm_FILES_TO_COPY_TO_FIXam) - 1 + FIXgsm_FILES_TO_COPY_TO_FIXam[i]=f"{fixgsm_ozone_fn}" + # + #----------------------------------------------------------------------- + # + # Set the element in the array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING that + # specifies the mapping between the symlink for the ozone production/loss + # file that must be created in each cycle directory and its target in the + # FIXam directory. The name of the symlink is alrady in the array, but + # the target is not because it depends on the ozone parameterization that + # the physics suite uses. Since we determined the ozone parameterization + # above, we now set the target of the symlink accordingly. + # + #----------------------------------------------------------------------- + # + ozone_symlink="global_o3prdlos.f77" + fixgsm_ozone_fn_is_set=False + regex_search="^[ ]*([^| ]*)[ ]*[|][ ]*([^| ]*)[ ]*$" + num_symlinks=len(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING) + + for i in range(num_symlinks): + mapping=CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[i] + symlink = find_pattern_in_str(regex_search, mapping) + if symlink is not None: + symlink = symlink[0] + if symlink == ozone_symlink: + regex_search="^[ ]*([^| ]+[ ]*)[|][ ]*([^| ]*)[ ]*$" + mapping_ozone = find_pattern_in_str(regex_search, mapping)[0] + mapping_ozone=f"{mapping_ozone}| {fixgsm_ozone_fn}" + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[i]=f"{mapping_ozone}" + fixgsm_ozone_fn_is_set=True + break + # + #----------------------------------------------------------------------- + # + # If fixgsm_ozone_fn_is_set is set to True, then the appropriate element + # of the array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING was set successfully. + # In this case, print out the new version of this array. Otherwise, print + # out an error message and exit. + # + #----------------------------------------------------------------------- + # + if fixgsm_ozone_fn_is_set: + + msg=dedent(f''' + After setting the file name of the ozone production/loss file in the + FIXgsm directory (based on the ozone parameterization specified in the + CCPP suite definition file), the array specifying the mapping between + the symlinks that need to be created in the cycle directories and the + files in the FIXam directory is: + + ''') + msg+=dedent(f''' + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} + ''') + print_info_msg(msg,verbose=VERBOSE) + + else: + + print_err_msg_exit(f''' + Unable to set name of the ozone production/loss file in the FIXgsm directory + in the array that specifies the mapping between the symlinks that need to + be created in the cycle directories and the files in the FIXgsm directory: + fixgsm_ozone_fn_is_set = \"{fixgsm_ozone_fn_is_set}\"''') + + EXPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] + export_vars(env_vars=EXPORTS) + + return ozone_param + +class Testing(unittest.TestCase): + def test_set_ozone_param(self): + self.assertEqual( "ozphys_2015", + set_ozone_param(ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml") ) + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ + "aerosol.dat | global_climaeropac_global.txt", + "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", + "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", + "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", + "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", + "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", + "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", + "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", + "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", + "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", + "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", + "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", + "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", + "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", + "co2monthlycyc.txt | co2monthlycyc.txt", + "global_h2oprdlos.f77 | global_h2o_pltc.f77", + "global_zorclim.1x1.grb | global_zorclim.1x1.grb", + "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", + "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", + "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77"] + FIXgsm_FILES_TO_COPY_TO_FIXam = [ + "global_glacier.2x2.grb", + "global_maxice.2x2.grb", + "RTGSST.1982.2012.monthly.clim.grb", + "global_snoclim.1.875.grb", + "CFSR.SEAICE.1982.2012.monthly.clim.grb", + "global_soilmgldas.t126.384.190.grb", + "seaice_newland.grb", + "global_climaeropac_global.txt", + "fix_co2_proj/global_co2historicaldata_2010.txt", + "fix_co2_proj/global_co2historicaldata_2011.txt", + "fix_co2_proj/global_co2historicaldata_2012.txt", + "fix_co2_proj/global_co2historicaldata_2013.txt", + "fix_co2_proj/global_co2historicaldata_2014.txt", + "fix_co2_proj/global_co2historicaldata_2015.txt", + "fix_co2_proj/global_co2historicaldata_2016.txt", + "fix_co2_proj/global_co2historicaldata_2017.txt", + "fix_co2_proj/global_co2historicaldata_2018.txt", + "fix_co2_proj/global_co2historicaldata_2019.txt", + "fix_co2_proj/global_co2historicaldata_2020.txt", + "fix_co2_proj/global_co2historicaldata_2021.txt", + "global_co2historicaldata_glob.txt", + "co2monthlycyc.txt", + "global_h2o_pltc.f77", + "global_hyblev.l65.txt", + "global_zorclim.1x1.grb", + "global_sfc_emissivity_idx.txt", + "global_solarconstant_noaa_an.txt", + "geo_em.d01.lat-lon.2.5m.HGT_M.nc", + "HGT.Beljaars_filtered.lat-lon.30s_res.nc", + "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77"] + + set_env_var('CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING', CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING) + set_env_var('FIXgsm_FILES_TO_COPY_TO_FIXam', FIXgsm_FILES_TO_COPY_TO_FIXam) diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py new file mode 100644 index 0000000000..e8f4e94ab3 --- /dev/null +++ b/ush/set_predef_grid_params.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import unittest +import os + +from constants import radius_Earth,degs_per_radian + +from python_utils import process_args,import_vars,export_vars,set_env_var,get_env_var,\ + print_input_args,define_macos_utilities, load_config_file, \ + cfg_to_yaml_str + +def set_predef_grid_params(): + """ Sets grid parameters for the specified predfined grid + + Args: + None + Returns: + None + """ + # import all environement variables + import_vars() + + params_dict = load_config_file("predef_grid_params.yaml") + params_dict = params_dict[PREDEF_GRID_NAME] + + # take care of special vars + special_vars = ['DT_ATMOS', 'LAYOUT_X', 'LAYOUT_Y', 'BLOCKSIZE'] + for var in special_vars: + if globals()[var] is not None: + params_dict[var] = globals()[var] + + #export variables to environment + export_vars(source_dict=params_dict) +# +#----------------------------------------------------------------------- +# +# Call the function defined above. +# +#----------------------------------------------------------------------- +# +if __name__ == "__main__": + set_predef_grid_params() + +class Testing(unittest.TestCase): + def test_set_predef_grid_params(self): + set_predef_grid_params() + self.assertEqual(get_env_var('GRID_GEN_METHOD'),"ESGgrid") + self.assertEqual(get_env_var('ESGgrid_LON_CTR'),-97.5) + + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG',False) + set_env_var('PREDEF_GRID_NAME',"RRFS_CONUS_3km") + set_env_var('DT_ATMOS',36) + set_env_var('LAYOUT_X',18) + set_env_var('LAYOUT_Y',36) + set_env_var('BLOCKSIZE',28) + set_env_var('QUILTING',True) + diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py new file mode 100644 index 0000000000..9a1703e098 --- /dev/null +++ b/ush/set_thompson_mp_fix_files.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 + +import os +import unittest +from textwrap import dedent + +from python_utils import import_vars,export_vars,set_env_var,list_to_str,\ + print_input_args,print_info_msg, print_err_msg_exit,\ + define_macos_utilities,load_xml_file,has_tag_with_value + +def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): + """ Function that first checks whether the Thompson + microphysics parameterization is being called by the selected physics + suite. If not, it sets the output variable whose name is specified by + output_varname_sdf_uses_thompson_mp to "FALSE" and exits. If so, it + sets this variable to "TRUE" and modifies the workflow arrays + FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + to ensure that fixed files needed by the Thompson microphysics + parameterization are copied to the FIXam directory and that appropriate + symlinks to these files are created in the run directories. + + Args: + ccpp_phys_suite_fp: full path to CCPP physics suite + thompson_mp_climo_fn: netcdf file for thompson microphysics + Returns: + boolean: sdf_uses_thompson_mp + """ + + print_input_args(locals()) + + # import all environment variables + import_vars() + + # + #----------------------------------------------------------------------- + # + # Check the suite definition file to see whether the Thompson microphysics + # parameterization is being used. + # + #----------------------------------------------------------------------- + # + tree = load_xml_file(ccpp_phys_suite_fp) + sdf_uses_thompson_mp = has_tag_with_value(tree, "scheme", "mp_thompson") + # + #----------------------------------------------------------------------- + # + # If the Thompson microphysics parameterization is being used, then... + # + #----------------------------------------------------------------------- + # + if sdf_uses_thompson_mp: + # + #----------------------------------------------------------------------- + # + # Append the names of the fixed files needed by the Thompson microphysics + # parameterization to the workflow array FIXgsm_FILES_TO_COPY_TO_FIXam, + # and append to the workflow array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + # the mappings between these files and the names of the corresponding + # symlinks that need to be created in the run directories. + # + #----------------------------------------------------------------------- + # + thompson_mp_fix_files=[ + "CCN_ACTIVATE.BIN", + "freezeH2O.dat", + "qr_acr_qg.dat", + "qr_acr_qs.dat", + "qr_acr_qgV2.dat", + "qr_acr_qsV2.dat" + ] + + if (EXTRN_MDL_NAME_ICS != "HRRR" and EXTRN_MDL_NAME_ICS != "RAP") or \ + (EXTRN_MDL_NAME_LBCS != "HRRR" and EXTRN_MDL_NAME_LBCS != "RAP"): + thompson_mp_fix_files.append(thompson_mp_climo_fn) + + FIXgsm_FILES_TO_COPY_TO_FIXam.extend(thompson_mp_fix_files) + + for fix_file in thompson_mp_fix_files: + mapping=f"{fix_file} | {fix_file}" + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING.append(mapping) + + msg=dedent(f''' + Since the Thompson microphysics parameterization is being used by this + physics suite (CCPP_PHYS_SUITE), the names of the fixed files needed by + this scheme have been appended to the array FIXgsm_FILES_TO_COPY_TO_FIXam, + and the mappings between these files and the symlinks that need to be + created in the cycle directories have been appended to the array + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the + values of these parameters are as follows: + + ''') + msg+=dedent(f''' + CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" + + FIXgsm_FILES_TO_COPY_TO_FIXam = {list_to_str(FIXgsm_FILES_TO_COPY_TO_FIXam)} + ''') + msg+=dedent(f''' + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} + ''') + print_info_msg(msg) + + EXPORTS = [ "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam" ] + export_vars(env_vars=EXPORTS) + + return sdf_uses_thompson_mp + +class Testing(unittest.TestCase): + def test_set_thompson_mp_fix_files(self): + self.assertEqual( True, + set_thompson_mp_fix_files(ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml", + thompson_mp_climo_fn="Thompson_MP_MONTHLY_CLIMO.nc") ) + def setUp(self): + define_macos_utilities(); + set_env_var('DEBUG',True) + set_env_var('VERBOSE',True) + set_env_var('EXTRN_MDL_NAME_ICS',"FV3GFS") + set_env_var('EXTRN_MDL_NAME_LBCS',"FV3GFS") + set_env_var('CCPP_PHYS_SUITE',"FV3_GSD_SAR") + + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ + "aerosol.dat | global_climaeropac_global.txt", + "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", + "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", + "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", + "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", + "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", + "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", + "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", + "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", + "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", + "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", + "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", + "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", + "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", + "co2monthlycyc.txt | co2monthlycyc.txt", + "global_h2oprdlos.f77 | global_h2o_pltc.f77", + "global_zorclim.1x1.grb | global_zorclim.1x1.grb", + "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", + "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", + "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77"] + + FIXgsm_FILES_TO_COPY_TO_FIXam = [ + "global_glacier.2x2.grb", + "global_maxice.2x2.grb", + "RTGSST.1982.2012.monthly.clim.grb", + "global_snoclim.1.875.grb", + "CFSR.SEAICE.1982.2012.monthly.clim.grb", + "global_soilmgldas.t126.384.190.grb", + "seaice_newland.grb", + "global_climaeropac_global.txt", + "fix_co2_proj/global_co2historicaldata_2010.txt", + "fix_co2_proj/global_co2historicaldata_2011.txt", + "fix_co2_proj/global_co2historicaldata_2012.txt", + "fix_co2_proj/global_co2historicaldata_2013.txt", + "fix_co2_proj/global_co2historicaldata_2014.txt", + "fix_co2_proj/global_co2historicaldata_2015.txt", + "fix_co2_proj/global_co2historicaldata_2016.txt", + "fix_co2_proj/global_co2historicaldata_2017.txt", + "fix_co2_proj/global_co2historicaldata_2018.txt", + "fix_co2_proj/global_co2historicaldata_2019.txt", + "fix_co2_proj/global_co2historicaldata_2020.txt", + "fix_co2_proj/global_co2historicaldata_2021.txt", + "global_co2historicaldata_glob.txt", + "co2monthlycyc.txt", + "global_h2o_pltc.f77", + "global_hyblev.l65.txt", + "global_zorclim.1x1.grb", + "global_sfc_emissivity_idx.txt", + "global_solarconstant_noaa_an.txt", + "geo_em.d01.lat-lon.2.5m.HGT_M.nc", + "HGT.Beljaars_filtered.lat-lon.30s_res.nc", + "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77"] + + set_env_var('CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING', CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING) + set_env_var('FIXgsm_FILES_TO_COPY_TO_FIXam', FIXgsm_FILES_TO_COPY_TO_FIXam) + diff --git a/ush/setup.py b/ush/setup.py new file mode 100644 index 0000000000..3584656ebf --- /dev/null +++ b/ush/setup.py @@ -0,0 +1,2195 @@ +#!/usr/bin/env python3 + +import os +import sys +import datetime +from textwrap import dedent + +from python_utils import cd_vrfy, mkdir_vrfy, rm_vrfy, check_var_valid_value,\ + lowercase,uppercase,check_for_preexist_dir_file,\ + list_to_str, type_to_str, \ + import_vars, export_vars, get_env_var, print_info_msg,\ + print_err_msg_exit, load_config_file, cfg_to_shell_str,\ + load_shell_config, load_ini_config, get_ini_value + +from set_cycle_dates import set_cycle_dates +from set_predef_grid_params import set_predef_grid_params +from set_ozone_param import set_ozone_param +from set_extrn_mdl_params import set_extrn_mdl_params +from set_gridparams_ESGgrid import set_gridparams_ESGgrid +from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid +from link_fix import link_fix +from check_ruc_lsm import check_ruc_lsm +from set_thompson_mp_fix_files import set_thompson_mp_fix_files + +def setup(): + """ Function that sets a secondary set + of parameters needed by the various scripts that are called by the + FV3-LAM rocoto community workflow. This secondary set of parameters is + calculated using the primary set of user-defined parameters in the de- + fault and custom experiment/workflow configuration scripts (whose file + names are defined below). This script then saves both sets of parame- + ters in a global variable definitions file (really a bash script) in + the experiment directory. This file then gets sourced by the various + scripts called by the tasks in the workflow. + + Args: + None + Returns: + None + """ + + ushdir=os.path.dirname(os.path.abspath(__file__)) + cd_vrfy(ushdir) + + # import all environment variables + import_vars() + + # print message + print_info_msg(f''' + ======================================================================== + Starting function setup() in \"{os.path.basename(__file__)}\"... + ========================================================================''') + # + #----------------------------------------------------------------------- + # + # Set the name of the configuration file containing default values for + # the experiment/workflow variables. Then source the file. + # + #----------------------------------------------------------------------- + # + EXPT_DEFAULT_CONFIG_FN="config_defaults.yaml" + cfg_d = load_config_file(EXPT_DEFAULT_CONFIG_FN) + import_vars(dictionary=cfg_d) + # + #----------------------------------------------------------------------- + # + # If a user-specified configuration file exists, source it. This file + # contains user-specified values for a subset of the experiment/workflow + # variables that override their default values. Note that the user- + # specified configuration file is not tracked by the repository, whereas + # the default configuration file is tracked. + # + #----------------------------------------------------------------------- + # + if os.path.exists(EXPT_CONFIG_FN): + # + # We require that the variables being set in the user-specified configu- + # ration file have counterparts in the default configuration file. This + # is so that we do not introduce new variables in the user-specified + # configuration file without also officially introducing them in the de- + # fault configuration file. Thus, before sourcing the user-specified + # configuration file, we check that all variables in the user-specified + # configuration file are also assigned default values in the default + # configuration file. + # + cfg_u = load_config_file(os.path.join(ushdir,EXPT_CONFIG_FN)) + cfg_d.update(cfg_u) + if cfg_u.items() > cfg_d.items(): + print_err_msg_exit(f''' + User specified config file {EXPT_CONGIG_FN} has variables that are + not defined in the default configuration file {EXPT_DEFAULT_CONFIG_FN}''') + import_vars(dictionary=cfg_u) + + # + #----------------------------------------------------------------------- + # + # If PREDEF_GRID_NAME is set to a non-empty string, set or reset parameters + # according to the predefined domain specified. + # + #----------------------------------------------------------------------- + # + + # export env vars before calling another module + export_vars() + + if PREDEF_GRID_NAME: + set_predef_grid_params() + + import_vars() + + # + #----------------------------------------------------------------------- + # + # Make sure different variables are set to their corresponding valid value + # + #----------------------------------------------------------------------- + # + global VERBOSE + if DEBUG and not VERBOSE: + print_info_msg(''' + Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"...''') + VERBOSE=False + + # + #----------------------------------------------------------------------- + # + # Set magnitude of stochastic ad-hoc schemes to -999.0 if they are not + # being used. This is required at the moment, since "do_shum/sppt/skeb" + # does not override the use of the scheme unless the magnitude is also + # specifically set to -999.0. If all "do_shum/sppt/skeb" are set to + # "false," then none will run, regardless of the magnitude values. + # + #----------------------------------------------------------------------- + # + global SHUM_MAG, SKEB_MAG, SPPT_MAG + if not DO_SHUM: + SHUM_MAG=-999.0 + if not DO_SKEB: + SKEB_MAG=-999.0 + if not DO_SPPT: + SPPT_MAG=-999.0 + # + #----------------------------------------------------------------------- + # + # If running with SPP in MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or + # RRTMG, count the number of entries in SPP_VAR_LIST to correctly set + # N_VAR_SPP, otherwise set it to zero. + # + #----------------------------------------------------------------------- + # + global N_VAR_SPP + N_VAR_SPP=0 + if DO_SPP: + N_VAR_SPP = len(SPP_VAR_LIST) + # + #----------------------------------------------------------------------- + # + # If running with Noah or RUC-LSM SPP, count the number of entries in + # LSM_SPP_VAR_LIST to correctly set N_VAR_LNDP, otherwise set it to zero. + # Also set LNDP_TYPE to 2 for LSM SPP, otherwise set it to zero. Finally, + # initialize an "FHCYC_LSM_SPP" variable to 0 and set it to 999 if LSM SPP + # is turned on. This requirement is necessary since LSM SPP cannot run with + # FHCYC=0 at the moment, but FHCYC cannot be set to anything less than the + # length of the forecast either. A bug fix will be submitted to + # ufs-weather-model soon, at which point, this requirement can be removed + # from regional_workflow. + # + #----------------------------------------------------------------------- + # + global N_VAR_LNDP, LNDP_TYPE, FHCYC_LSM_SPP_OR_NOT + N_VAR_LNDP=0 + LNDP_TYPE=0 + FHCYC_LSM_SPP_OR_NOT=0 + if DO_LSM_SPP: + N_VAR_LNDP=len(LSM_SPP_VAR_LIST) + LNDP_TYPE=2 + FHCYC_LSM_SPP_OR_NOT=999 + # + #----------------------------------------------------------------------- + # + # If running with SPP, confirm that each SPP-related namelist value + # contains the same number of entries as N_VAR_SPP (set above to be equal + # to the number of entries in SPP_VAR_LIST). + # + #----------------------------------------------------------------------- + # + if DO_SPP: + if ( len(SPP_MAG_LIST) != N_VAR_SPP ) or \ + ( len(SPP_LSCALE) != N_VAR_SPP) or \ + ( len(SPP_TSCALE) != N_VAR_SPP) or \ + ( len(SPP_SIGTOP1) != N_VAR_SPP) or \ + ( len(SPP_SIGTOP2) != N_VAR_SPP) or \ + ( len(SPP_STDDEV_CUTOFF) != N_VAR_SPP) or \ + ( len(ISEED_SPP) != N_VAR_SPP): + print_err_msg_exit(f''' + All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist + variables set in {CONFIG_FN} must be equal in number of entries to what is + found in SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"{len(SPP_VAR_LIST)}\"''') + # + #----------------------------------------------------------------------- + # + # If running with LSM SPP, confirm that each LSM SPP-related namelist + # value contains the same number of entries as N_VAR_LNDP (set above to + # be equal to the number of entries in LSM_SPP_VAR_LIST). + # + #----------------------------------------------------------------------- + # + if DO_LSM_SPP: + if ( len(LSM_SPP_MAG_LIST) != N_VAR_LNDP) or \ + ( len(LSM_SPP_LSCALE) != N_VAR_LNDP) or \ + ( len(LSM_SPP_TSCALE) != N_VAR_LNDP): + print_err_msg_exit(f''' + All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) + set in {CONFIG_FN} must be equal in number of entries to what is found in + SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"{len(LSM_SPP_VAR_LIST)}\"''') + # + # The current script should be located in the ush subdirectory of the + # workflow directory. Thus, the workflow directory is the one above the + # directory of the current script. + # + SR_WX_APP_TOP_DIR=os.path.abspath( os.path.dirname(__file__) + \ + os.sep + os.pardir + os.sep + os.pardir) + + # + #----------------------------------------------------------------------- + # + # Set the base directories in which codes obtained from external reposi- + # tories (using the manage_externals tool) are placed. Obtain the rela- + # tive paths to these directories by reading them in from the manage_ex- + # ternals configuration file. (Note that these are relative to the lo- + # cation of the configuration file.) Then form the full paths to these + # directories. Finally, make sure that each of these directories actu- + # ally exists. + # + #----------------------------------------------------------------------- + # + mng_extrns_cfg_fn = os.path.join(SR_WX_APP_TOP_DIR, "Externals.cfg") + try: + mng_extrns_cfg_fn = os.readlink(mng_extrns_cfg_fn) + except: + pass + property_name="local_path" + cfg = load_ini_config(mng_extrns_cfg_fn) + # + # Get the path to the workflow scripts + # + external_name="regional_workflow" + HOMErrfs = get_ini_value(cfg, external_name, property_name) + + if not HOMErrfs: + print_err_msg_exit(f''' + Externals.cfg does not contain "{external_name}".''') + + HOMErrfs = os.path.join(SR_WX_APP_TOP_DIR, HOMErrfs) + # + # Get the base directory of the FV3 forecast model code. + # + external_name=FCST_MODEL + UFS_WTHR_MDL_DIR = get_ini_value(cfg, external_name,property_name) + + if not UFS_WTHR_MDL_DIR: + print_err_msg_exit(f''' + Externals.cfg does not contain "{external_name}".''') + + UFS_WTHR_MDL_DIR=os.path.join(SR_WX_APP_TOP_DIR, UFS_WTHR_MDL_DIR) + if not os.path.exists(UFS_WTHR_MDL_DIR): + print_err_msg_exit(f''' + The base directory in which the FV3 source code should be located + (UFS_WTHR_MDL_DIR) does not exist: + UFS_WTHR_MDL_DIR = \"{UFS_WTHR_MDL_DIR}\" + Please clone the external repository containing the code in this directory, + build the executable, and then rerun the workflow.''') + # + # Get the base directory of the UFS_UTILS codes. + # + external_name="ufs_utils" + UFS_UTILS_DIR=get_ini_value(cfg, external_name, property_name) + + if not UFS_UTILS_DIR: + print_err_msg_exit(f''' + Externals.cfg does not contain "{external_name}".''') + + UFS_UTILS_DIR=os.path.join(SR_WX_APP_TOP_DIR, UFS_UTILS_DIR) + if not os.path.exists(UFS_UTILS_DIR): + print_err_msg_exit(f''' + The base directory in which the UFS utilities source codes should be lo- + cated (UFS_UTILS_DIR) does not exist: + UFS_UTILS_DIR = \"{UFS_UTILS_DIR}\" + Please clone the external repository containing the code in this direct- + ory, build the executables, and then rerun the workflow.''') + # + # Get the base directory of the UPP code. + # + external_name="UPP" + UPP_DIR=get_ini_value(cfg,external_name,property_name ) + if not UPP_DIR: + print_err_msg_exit(f''' + Externals.cfg does not contain "{external_name}".''') + + UPP_DIR=os.path.join(SR_WX_APP_TOP_DIR, UPP_DIR) + if not os.path.exists(UPP_DIR): + print_err_msg_exit(f''' + The base directory in which the UPP source code should be located + (UPP_DIR) does not exist: + UPP_DIR = \"{UPP_DIR}\" + Please clone the external repository containing the code in this directory, + build the executable, and then rerun the workflow.''') + + # + # Define some other useful paths + # + global USHDIR, SCRIPTSDIR, JOBSDIR,SORCDIR, SRC_DIR, PARMDIR, MODULES_DIR, EXECDIR, TEMPLATE_DIR, \ + VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG + + USHDIR = os.path.join(HOMErrfs,"ush") + SCRIPTSDIR = os.path.join(HOMErrfs,"scripts") + JOBSDIR = os.path.join(HOMErrfs,"jobs") + SORCDIR = os.path.join(HOMErrfs,"sorc") + SRC_DIR = os.path.join(SR_WX_APP_TOP_DIR,"src") + PARMDIR = os.path.join(HOMErrfs,"parm") + MODULES_DIR = os.path.join(HOMErrfs,"modulefiles") + EXECDIR = os.path.join(SR_WX_APP_TOP_DIR,"bin") + TEMPLATE_DIR = os.path.join(USHDIR,"templates") + VX_CONFIG_DIR = os.path.join(TEMPLATE_DIR,"parm") + METPLUS_CONF = os.path.join(TEMPLATE_DIR,"parm","metplus") + MET_CONFIG = os.path.join(TEMPLATE_DIR,"parm","met") + + # + #----------------------------------------------------------------------- + # + # Source the machine config file containing architechture information, + # queue names, and supported input file paths. + # + #----------------------------------------------------------------------- + # + global MACHINE + global MACHINE_FILE + global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, FIXLAM_NCO_BASEDIR, \ + RELATIVE_LINK_FLAG, WORKFLOW_MANAGER, NCORES_PER_NODE, SCHED, \ + QUEUE_DEFAULT, QUEUE_HPSS, QUEUE_FCST, \ + PARTITION_DEFAULT, PARTITION_HPSS, PARTITION_FCST + + MACHINE = uppercase(MACHINE) + RELATIVE_LINK_FLAG="--relative" + MACHINE_FILE=MACHINE_FILE or os.path.join(USHDIR,"machine",f"{lowercase(MACHINE)}.sh") + machine_cfg = load_shell_config(MACHINE_FILE) + import_vars(dictionary=machine_cfg) + + if not NCORES_PER_NODE: + print_err_msg_exit(f""" + NCORES_PER_NODE has not been specified in the file {MACHINE_FILE} + Please ensure this value has been set for your desired platform. """) + + if not (FIXgsm and FIXaer and FIXlut and TOPO_DIR and SFC_CLIMO_INPUT_DIR): + print_err_msg_exit(f''' + One or more fix file directories have not been specified for this machine: + MACHINE = \"{MACHINE}\" + FIXgsm = \"{FIXgsm or ""} + FIXaer = \"{FIXaer or ""} + FIXlut = \"{FIXlut or ""} + TOPO_DIR = \"{TOPO_DIR or ""} + SFC_CLIMO_INPUT_DIR = \"{SFC_CLIMO_INPUT_DIR or ""} + FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR or ""} + You can specify the missing location(s) in config.sh''') + + # + #----------------------------------------------------------------------- + # + # Set the names of the build and workflow environment files (if not + # already specified by the user). These are the files that need to be + # sourced before building the component SRW App codes and running various + # workflow scripts, respectively. + # + #----------------------------------------------------------------------- + # + global WFLOW_ENV_FN, BUILD_ENV_FN + machine=lowercase(MACHINE) + WFLOW_ENV_FN=WFLOW_ENV_FN or f"wflow_{machine}.env" + BUILD_ENV_FN=BUILD_ENV_FN or f"build_{machine}_{COMPILER}.env" + # + #----------------------------------------------------------------------- + # + # Calculate a default value for the number of processes per node for the + # RUN_FCST_TN task. Then set PPN_RUN_FCST to this default value if + # PPN_RUN_FCST is not already specified by the user. + # + #----------------------------------------------------------------------- + # + global PPN_RUN_FCST + ppn_run_fcst_default = NCORES_PER_NODE // OMP_NUM_THREADS_RUN_FCST + PPN_RUN_FCST=PPN_RUN_FCST or ppn_run_fcst_default + # + #----------------------------------------------------------------------- + # + # If we are using a workflow manager check that the ACCOUNT variable is + # not empty. + # + #----------------------------------------------------------------------- + # + if WORKFLOW_MANAGER != "none": + if not ACCOUNT: + print_err_msg_exit(f''' + The variable ACCOUNT cannot be empty if you are using a workflow manager: + ACCOUNT = \"ACCOUNT\" + WORKFLOW_MANAGER = \"WORKFLOW_MANAGER\"''') + # + #----------------------------------------------------------------------- + # + # Set the grid type (GTYPE). In general, in the FV3 code, this can take + # on one of the following values: "global", "stretch", "nest", and "re- + # gional". The first three values are for various configurations of a + # global grid, while the last one is for a regional grid. Since here we + # are only interested in a regional grid, GTYPE must be set to "region- + # al". + # + #----------------------------------------------------------------------- + # + global TILE_RGNL, GTYPE + GTYPE="regional" + TILE_RGNL="7" + + #----------------------------------------------------------------------- + # + # Set USE_MERRA_CLIMO to either "TRUE" or "FALSE" so we don't + # have to consider other valid values later on. + # + #----------------------------------------------------------------------- + global USE_MERRA_CLIMO + if USE_MERRA_CLIMO == "FV3_GFS_v15_thompson_mynn_lam3km": + USE_MERRA_CLIMO=True + # + #----------------------------------------------------------------------- + # + # Set CPL to TRUE/FALSE based on FCST_MODEL. + # + #----------------------------------------------------------------------- + # + global CPL + if FCST_MODEL == "ufs-weather-model": + CPL=False + elif FCST_MODEL == "fv3gfs_aqm": + CPL=True + else: + print_err_msg_exit(f''' + The coupling flag CPL has not been specified for this value of FCST_MODEL: + FCST_MODEL = \"{FCST_MODEL}\"''') + # + #----------------------------------------------------------------------- + # + # Make sure RESTART_INTERVAL is set to an integer value if present + # + #----------------------------------------------------------------------- + # + if not isinstance(RESTART_INTERVAL,int): + print_err_msg_exit(f''' + RESTART_INTERVAL must be set to an integer number of hours. + RESTART_INTERVAL = \"{RESTART_INTERVAL}\"''') + # + #----------------------------------------------------------------------- + # + # Check that DATE_FIRST_CYCL and DATE_LAST_CYCL are strings consisting + # of exactly 8 digits. + # + #----------------------------------------------------------------------- + # + if not isinstance(DATE_FIRST_CYCL,datetime.date): + print_err_msg_exit(f''' + DATE_FIRST_CYCL must be a string consisting of exactly 8 digits of the + form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit + month, and DD is the 2-digit day-of-month. + DATE_FIRST_CYCL = \"{DATE_FIRST_CYCL}\"''') + + if not isinstance(DATE_LAST_CYCL,datetime.date): + print_err_msg_exit(f''' + DATE_LAST_CYCL must be a string consisting of exactly 8 digits of the + form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit + month, and DD is the 2-digit day-of-month. + DATE_LAST_CYCL = \"{DATE_LAST_CYCL}\"''') + # + #----------------------------------------------------------------------- + # + # Check that all elements of CYCL_HRS are strings consisting of exactly + # 2 digits that are between "00" and "23", inclusive. + # + #----------------------------------------------------------------------- + # + i=0 + for CYCL in CYCL_HRS: + if CYCL < 0 or CYCL > 23: + print_err_msg_exit(f''' + Each element of CYCL_HRS must be an integer between \"00\" and \"23\", in- + clusive (including a leading \"0\", if necessary), specifying an hour-of- + day. Element #{i} of CYCL_HRS (where the index of the first element is 0) + does not have this form: + CYCL_HRS = {CYCL_HRS} + CYCL_HRS[{i}] = \"{CYCL_HRS[i]}\"''') + + i=i+1 + # + #----------------------------------------------------------------------- + # Check cycle increment for cycle frequency (cycl_freq). + # only if INCR_CYCL_FREQ < 24. + #----------------------------------------------------------------------- + # + if INCR_CYCL_FREQ < 24 and i > 1: + cycl_intv=(24//i) + if cycl_intv != INCR_CYCL_FREQ: + print_err_msg_exit(f''' + The number of CYCL_HRS does not match with that expected by INCR_CYCL_FREQ: + INCR_CYCL_FREQ = {INCR_CYCL_FREQ} + cycle interval by the number of CYCL_HRS = {cycl_intv} + CYCL_HRS = {CYCL_HRS} ''') + + for itmp in range(1,i): + itm1=itmp-1 + cycl_next_itmp=CYCL_HRS[itm1] + INCR_CYCL_FREQ + if cycl_next_itmp != CYCL_HRS[itmp]: + print_err_msg_exit(f''' + Element {itmp} of CYCL_HRS does not match with the increment of cycle + frequency INCR_CYCL_FREQ: + CYCL_HRS = {CYCL_HRS} + INCR_CYCL_FREQ = {INCR_CYCL_FREQ} + CYCL_HRS[{itmp}] = \"{CYCL_HRS[itmp]}\"''') + # + #----------------------------------------------------------------------- + # + # Call a function to generate the array ALL_CDATES containing the cycle + # dates/hours for which to run forecasts. The elements of this array + # will have the form YYYYMMDDHH. They are the starting dates/times of + # the forecasts that will be run in the experiment. Then set NUM_CYCLES + # to the number of elements in this array. + # + #----------------------------------------------------------------------- + # + + ALL_CDATES = set_cycle_dates( \ + date_start=DATE_FIRST_CYCL, + date_end=DATE_LAST_CYCL, + cycle_hrs=CYCL_HRS, + incr_cycl_freq=INCR_CYCL_FREQ) + + NUM_CYCLES=len(ALL_CDATES) + + if NUM_CYCLES > 90: + ALL_CDATES=None + print_info_msg(f''' + Too many cycles in ALL_CDATES to list, redefining in abbreviated form." + ALL_CDATES="{DATE_FIRST_CYCL}{CYCL_HRS[0]}...{DATE_LAST_CYCL}{CYCL_HRS[-1]}''') + # + #----------------------------------------------------------------------- + # + # If using a custom post configuration file, make sure that it exists. + # + #----------------------------------------------------------------------- + # + if USE_CUSTOM_POST_CONFIG_FILE: + if not os.path.exists(CUSTOM_POST_CONFIG_FP): + print_err_msg_exit(f''' + The custom post configuration specified by CUSTOM_POST_CONFIG_FP does not + exist: + CUSTOM_POST_CONFIG_FP = \"{CUSTOM_POST_CONFIG_FP}\"''') + # + #----------------------------------------------------------------------- + # + # If using external CRTM fix files to allow post-processing of synthetic + # satellite products from the UPP, then make sure the fix file directory + # exists. + # + #----------------------------------------------------------------------- + # + if USE_CRTM: + if not os.path.exists(CRTM_DIR): + print_err_msg_exit(f''' + The external CRTM fix file directory specified by CRTM_DIR does not exist: + CRTM_DIR = \"{CRTM_DIR}\"''') + # + #----------------------------------------------------------------------- + # + # The forecast length (in integer hours) cannot contain more than 3 cha- + # racters. Thus, its maximum value is 999. Check whether the specified + # forecast length exceeds this maximum value. If so, print out a warn- + # ing and exit this script. + # + #----------------------------------------------------------------------- + # + fcst_len_hrs_max=999 + if FCST_LEN_HRS > fcst_len_hrs_max: + print_err_msg_exit(f''' + Forecast length is greater than maximum allowed length: + FCST_LEN_HRS = {FCST_LEN_HRS} + fcst_len_hrs_max = {fcst_len_hrs_max}''') + # + #----------------------------------------------------------------------- + # + # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible + # by the BC update interval (LBC_SPEC_INTVL_HRS). If not, print out a + # warning and exit this script. If so, generate an array of forecast + # hours at which the boundary values will be updated. + # + #----------------------------------------------------------------------- + # + rem=FCST_LEN_HRS%LBC_SPEC_INTVL_HRS + + if rem != 0: + print_err_msg_exit(f''' + The forecast length (FCST_LEN_HRS) is not evenly divisible by the lateral + boundary conditions update interval (LBC_SPEC_INTVL_HRS): + FCST_LEN_HRS = {FCST_LEN_HRS} + LBC_SPEC_INTVL_HRS = {LBC_SPEC_INTVL_HRS} + rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = {rem}''') + # + #----------------------------------------------------------------------- + # + # Set the array containing the forecast hours at which the lateral + # boundary conditions (LBCs) need to be updated. Note that this array + # does not include the 0-th hour (initial time). + # + #----------------------------------------------------------------------- + # + LBC_SPEC_FCST_HRS=[ i for i in range(LBC_SPEC_INTVL_HRS, \ + LBC_SPEC_INTVL_HRS + FCST_LEN_HRS, \ + LBC_SPEC_INTVL_HRS ) ] + # + #----------------------------------------------------------------------- + # + # Check to make sure that various computational parameters needed by the + # forecast model are set to non-empty values. At this point in the + # experiment generation, all of these should be set to valid (non-empty) + # values. + # + #----------------------------------------------------------------------- + # + if not DT_ATMOS: + print_err_msg_exit(f''' + The forecast model main time step (DT_ATMOS) is set to a null string: + DT_ATMOS = {DT_ATMOS} + Please set this to a valid numerical value in the user-specified experiment + configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''') + + if not LAYOUT_X: + print_err_msg_exit(f''' + The number of MPI processes to be used in the x direction (LAYOUT_X) by + the forecast job is set to a null string: + LAYOUT_X = {LAYOUT_X} + Please set this to a valid numerical value in the user-specified experiment + configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''') + + if not LAYOUT_Y: + print_err_msg_exit(f''' + The number of MPI processes to be used in the y direction (LAYOUT_Y) by + the forecast job is set to a null string: + LAYOUT_Y = {LAYOUT_Y} + Please set this to a valid numerical value in the user-specified experiment + configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''') + + if not BLOCKSIZE: + print_err_msg_exit(f''' + The cache size to use for each MPI task of the forecast (BLOCKSIZE) is + set to a null string: + BLOCKSIZE = {BLOCKSIZE} + Please set this to a valid numerical value in the user-specified experiment + configuration file (EXPT_CONFIG_FP) and rerun: + EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''') + # + #----------------------------------------------------------------------- + # + # If performing sub-hourly model output and post-processing, check that + # the output interval DT_SUBHOURLY_POST_MNTS (in minutes) is specified + # correctly. + # + #----------------------------------------------------------------------- + # + global SUB_HOURLY_POST + + if SUB_HOURLY_POST: + # + # Check that DT_SUBHOURLY_POST_MNTS is between 0 and 59, inclusive. + # + if DT_SUBHOURLY_POST_MNTS < 0 or DT_SUBHOURLY_POST_MNTS > 59: + print_err_msg_exit(f''' + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + DT_SUBHOURLY_POST_MNTS must be set to an integer between 0 and 59, + inclusive but in this case is not: + SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" + DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\"''') + # + # Check that DT_SUBHOURLY_POST_MNTS (after converting to seconds) is + # evenly divisible by the forecast model's main time step DT_ATMOS. + # + rem=( DT_SUBHOURLY_POST_MNTS*60 % DT_ATMOS ) + if rem != 0: + print_err_msg_exit(f''' + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + the time interval specified by DT_SUBHOURLY_POST_MNTS (after converting + to seconds) must be evenly divisible by the time step DT_ATMOS used in + the forecast model, i.e. the remainder (rem) must be zero. In this case, + it is not: + SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" + DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" + DT_ATMOS = \"{DT_ATMOS}\" + rem = (DT_SUBHOURLY_POST_MNTS*60) %% DT_ATMOS = {rem} + Please reset DT_SUBHOURLY_POST_MNTS and/or DT_ATMOS so that this remainder + is zero.''') + # + # If DT_SUBHOURLY_POST_MNTS is set to 0 (with SUB_HOURLY_POST set to + # True), then we're not really performing subhourly post-processing. + # In this case, reset SUB_HOURLY_POST to False and print out an + # informational message that such a change was made. + # + if DT_SUBHOURLY_POST_MNTS == 0: + print_info_msg(f''' + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + DT_SUBHOURLY_POST_MNTS must be set to a value greater than 0; otherwise, + sub-hourly output is not really being performed: + SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" + DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" + Resetting SUB_HOURLY_POST to \"FALSE\". If you do not want this, you + must set DT_SUBHOURLY_POST_MNTS to something other than zero.''') + SUB_HOURLY_POST=False + # + #----------------------------------------------------------------------- + # + # If the base directory (EXPT_BASEDIR) in which the experiment subdirectory + # (EXPT_SUBDIR) will be located does not start with a "/", then it is + # either set to a null string or contains a relative directory. In both + # cases, prepend to it the absolute path of the default directory under + # which the experiment directories are placed. If EXPT_BASEDIR was set + # to a null string, it will get reset to this default experiment directory, + # and if it was set to a relative directory, it will get reset to an + # absolute directory that points to the relative directory under the + # default experiment directory. Then create EXPT_BASEDIR if it doesn't + # already exist. + # + #----------------------------------------------------------------------- + # + global EXPT_BASEDIR + if (not EXPT_BASEDIR) or (EXPT_BASEDIR[0] != "/"): + if not EXPT_BASEDIR: + EXPT_BASEDIR = "" + EXPT_BASEDIR = os.path.join(SR_WX_APP_TOP_DIR,"..","expt_dirs",EXPT_BASEDIR) + try: + EXPT_BASEDIR = os.path.realpath(EXPT_BASEDIR) + except: + pass + EXPT_BASEDIR = os.path.abspath(EXPT_BASEDIR) + + mkdir_vrfy(f' -p "{EXPT_BASEDIR}"') + # + #----------------------------------------------------------------------- + # + # If the experiment subdirectory name (EXPT_SUBDIR) is set to an empty + # string, print out an error message and exit. + # + #----------------------------------------------------------------------- + # + if not EXPT_SUBDIR: + print_err_msg_exit(f''' + The name of the experiment subdirectory (EXPT_SUBDIR) cannot be empty: + EXPT_SUBDIR = \"{EXPT_SUBDIR}\"''') + # + #----------------------------------------------------------------------- + # + # Set the full path to the experiment directory. Then check if it already + # exists and if so, deal with it as specified by PREEXISTING_DIR_METHOD. + # + #----------------------------------------------------------------------- + # + global EXPTDIR + EXPTDIR = os.path.join(EXPT_BASEDIR, EXPT_SUBDIR) + check_for_preexist_dir_file(EXPTDIR,PREEXISTING_DIR_METHOD) + # + #----------------------------------------------------------------------- + # + # Set other directories, some of which may depend on EXPTDIR (depending + # on whether we're running in NCO or community mode, i.e. whether RUN_ENVIR + # is set to "nco" or "community"). Definitions: + # + # LOGDIR: + # Directory in which the log files from the workflow tasks will be placed. + # + # FIXam: + # This is the directory that will contain the fixed files or symlinks to + # the fixed files containing various fields on global grids (which are + # usually much coarser than the native FV3-LAM grid). + # + # FIXclim: + # This is the directory that will contain the MERRA2 aerosol climatology + # data file and lookup tables for optics properties + # + # FIXLAM: + # This is the directory that will contain the fixed files or symlinks to + # the fixed files containing the grid, orography, and surface climatology + # on the native FV3-LAM grid. + # + # CYCLE_BASEDIR: + # The base directory in which the directories for the various cycles will + # be placed. + # + # COMROOT: + # In NCO mode, this is the full path to the "com" directory under which + # output from the RUN_POST_TN task will be placed. Note that this output + # is not placed directly under COMROOT but several directories further + # down. More specifically, for a cycle starting at yyyymmddhh, it is at + # + # $COMROOT/$NET/$envir/$RUN.$yyyymmdd/$hh + # + # Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMOROOT + # is not used by the workflow in community mode. + # + # COMOUT_BASEDIR: + # In NCO mode, this is the base directory directly under which the output + # from the RUN_POST_TN task will be placed, i.e. it is the cycle-independent + # portion of the RUN_POST_TN task's output directory. It is given by + # + # $COMROOT/$NET/$model_ver + # + # COMOUT_BASEDIR is not used by the workflow in community mode. + # + #----------------------------------------------------------------------- + # + global LOGDIR, FIXam, FIXclim, FIXLAM, CYCLE_BASEDIR, \ + COMROOT, COMOUT_BASEDIR + + LOGDIR = os.path.join(EXPTDIR, "log") + + FIXam = os.path.join(EXPTDIR, "fix_am") + FIXclim = os.path.join(EXPTDIR, "fix_clim") + FIXLAM = os.path.join(EXPTDIR, "fix_lam") + + if RUN_ENVIR == "nco": + + CYCLE_BASEDIR = os.path.join(STMP, "tmpnwprd", RUN) + check_for_preexist_dir_file(CYCLE_BASEDIR,PREEXISTING_DIR_METHOD) + COMROOT = os.path.join(PTMP, "com") + COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) + check_for_preexist_dir_file(COMOUT_BASEDIR,PREEXISTING_DIR_METHOD) + + else: + + CYCLE_BASEDIR=EXPTDIR + COMROOT="" + COMOUT_BASEDIR="" + # + #----------------------------------------------------------------------- + # + # The FV3 forecast model needs the following input files in the run di- + # rectory to start a forecast: + # + # (1) The data table file + # (2) The diagnostics table file + # (3) The field table file + # (4) The FV3 namelist file + # (5) The model configuration file + # (6) The NEMS configuration file + # + # If using CCPP, it also needs: + # + # (7) The CCPP physics suite definition file + # + # The workflow contains templates for the first six of these files. + # Template files are versions of these files that contain placeholder + # (i.e. dummy) values for various parameters. The experiment/workflow + # generation scripts copy these templates to appropriate locations in + # the experiment directory (either the top of the experiment directory + # or one of the cycle subdirectories) and replace the placeholders in + # these copies by actual values specified in the experiment/workflow + # configuration file (or derived from such values). The scripts then + # use the resulting "actual" files as inputs to the forecast model. + # + # Note that the CCPP physics suite defintion file does not have a cor- + # responding template file because it does not contain any values that + # need to be replaced according to the experiment/workflow configura- + # tion. If using CCPP, this file simply needs to be copied over from + # its location in the forecast model's directory structure to the ex- + # periment directory. + # + # Below, we first set the names of the templates for the first six files + # listed above. We then set the full paths to these template files. + # Note that some of these file names depend on the physics suite while + # others do not. + # + #----------------------------------------------------------------------- + # + global DATA_TABLE_TMPL_FN, DIAG_TABLE_TMPL_FN, FIELD_TABLE_TMPL_FN, \ + MODEL_CONFIG_TMPL_FN, NEMS_CONFIG_TMPL_FN + global DATA_TABLE_TMPL_FP, DIAG_TABLE_TMPL_FP, FIELD_TABLE_TMPL_FP, \ + MODEL_CONFIG_TMPL_FP, NEMS_CONFIG_TMPL_FP + global FV3_NML_BASE_SUITE_FP, FV3_NML_YAML_CONFIG_FP,FV3_NML_BASE_ENS_FP + + dot_ccpp_phys_suite_or_null=f".{CCPP_PHYS_SUITE}" + + # Names of input files that the forecast model (ufs-weather-model) expects + # to read in. These should only be changed if the input file names in the + # forecast model code are changed. + #---------------------------------- + DATA_TABLE_FN = "data_table" + DIAG_TABLE_FN = "diag_table" + FIELD_TABLE_FN = "field_table" + MODEL_CONFIG_FN = "model_configure" + NEMS_CONFIG_FN = "nems.configure" + #---------------------------------- + + if DATA_TABLE_TMPL_FN is None: + DATA_TABLE_TMPL_FN = DATA_TABLE_FN + if DIAG_TABLE_TMPL_FN is None: + DIAG_TABLE_TMPL_FN = f"{DIAG_TABLE_FN}{dot_ccpp_phys_suite_or_null}" + if FIELD_TABLE_TMPL_FN is None: + FIELD_TABLE_TMPL_FN = f"{FIELD_TABLE_FN}{dot_ccpp_phys_suite_or_null}" + if MODEL_CONFIG_TMPL_FN is None: + MODEL_CONFIG_TMPL_FN = MODEL_CONFIG_FN + if NEMS_CONFIG_TMPL_FN is None: + NEMS_CONFIG_TMPL_FN = NEMS_CONFIG_FN + + DATA_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,DATA_TABLE_TMPL_FN) + DIAG_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,DIAG_TABLE_TMPL_FN) + FIELD_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,FIELD_TABLE_TMPL_FN) + FV3_NML_BASE_SUITE_FP = os.path.join(TEMPLATE_DIR,FV3_NML_BASE_SUITE_FN) + FV3_NML_YAML_CONFIG_FP = os.path.join(TEMPLATE_DIR,FV3_NML_YAML_CONFIG_FN) + FV3_NML_BASE_ENS_FP = os.path.join(EXPTDIR,FV3_NML_BASE_ENS_FN) + MODEL_CONFIG_TMPL_FP = os.path.join(TEMPLATE_DIR,MODEL_CONFIG_TMPL_FN) + NEMS_CONFIG_TMPL_FP = os.path.join(TEMPLATE_DIR,NEMS_CONFIG_TMPL_FN) + # + #----------------------------------------------------------------------- + # + # Set: + # + # 1) the variable CCPP_PHYS_SUITE_FN to the name of the CCPP physics + # suite definition file. + # 2) the variable CCPP_PHYS_SUITE_IN_CCPP_FP to the full path of this + # file in the forecast model's directory structure. + # 3) the variable CCPP_PHYS_SUITE_FP to the full path of this file in + # the experiment directory. + # + # Note that the experiment/workflow generation scripts will copy this + # file from CCPP_PHYS_SUITE_IN_CCPP_FP to CCPP_PHYS_SUITE_FP. Then, for + # each cycle, the forecast launch script will create a link in the cycle + # run directory to the copy of this file at CCPP_PHYS_SUITE_FP. + # + #----------------------------------------------------------------------- + # + global CCPP_PHYS_SUITE_FN, CCPP_PHYS_SUITE_IN_CCPP_FP, CCPP_PHYS_SUITE_FP + CCPP_PHYS_SUITE_FN=f"suite_{CCPP_PHYS_SUITE}.xml" + CCPP_PHYS_SUITE_IN_CCPP_FP=os.path.join(UFS_WTHR_MDL_DIR, "FV3","ccpp","suites",CCPP_PHYS_SUITE_FN) + CCPP_PHYS_SUITE_FP=os.path.join(EXPTDIR, CCPP_PHYS_SUITE_FN) + if not os.path.exists(CCPP_PHYS_SUITE_IN_CCPP_FP): + print_err_msg_exit(f''' + The CCPP suite definition file (CCPP_PHYS_SUITE_IN_CCPP_FP) does not exist + in the local clone of the ufs-weather-model: + CCPP_PHYS_SUITE_IN_CCPP_FP = \"{CCPP_PHYS_SUITE_IN_CCPP_FP}\"''') + # + #----------------------------------------------------------------------- + # + # Set: + # + # 1) the variable FIELD_DICT_FN to the name of the field dictionary + # file. + # 2) the variable FIELD_DICT_IN_UWM_FP to the full path of this + # file in the forecast model's directory structure. + # 3) the variable FIELD_DICT_FP to the full path of this file in + # the experiment directory. + # + #----------------------------------------------------------------------- + # + global FIELD_DICT_FN, FIELD_DICT_IN_UWM_FP, FIELD_DICT_FP + FIELD_DICT_FN = "fd_nems.yaml" + FIELD_DICT_IN_UWM_FP = os.path.join(UFS_WTHR_MDL_DIR, "tests", "parm", FIELD_DICT_FN) + FIELD_DICT_FP = os.path.join(EXPTDIR, FIELD_DICT_FN) + if not os.path.exists(FIELD_DICT_IN_UWM_FP): + print_err_msg_exit(f''' + The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist + in the local clone of the ufs-weather-model: + FIELD_DICT_IN_UWM_FP = \"{FIELD_DICT_IN_UWM_FP}\"''') + # + #----------------------------------------------------------------------- + # + # Call the function that sets the ozone parameterization being used and + # modifies associated parameters accordingly. + # + #----------------------------------------------------------------------- + # + + # export env vars before calling another module + export_vars() + + OZONE_PARAM = set_ozone_param( \ + ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP) + + IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] + import_vars(env_vars=IMPORTS) + # + #----------------------------------------------------------------------- + # + # Set the full paths to those forecast model input files that are cycle- + # independent, i.e. they don't include information about the cycle's + # starting day/time. These are: + # + # * The data table file [(1) in the list above)] + # * The field table file [(3) in the list above)] + # * The FV3 namelist file [(4) in the list above)] + # * The NEMS configuration file [(6) in the list above)] + # + # Since they are cycle-independent, the experiment/workflow generation + # scripts will place them in the main experiment directory (EXPTDIR). + # The script that runs each cycle will then create links to these files + # in the run directories of the individual cycles (which are subdirecto- + # ries under EXPTDIR). + # + # The remaining two input files to the forecast model, i.e. + # + # * The diagnostics table file [(2) in the list above)] + # * The model configuration file [(5) in the list above)] + # + # contain parameters that depend on the cycle start date. Thus, custom + # versions of these two files must be generated for each cycle and then + # placed directly in the run directories of the cycles (not EXPTDIR). + # For this reason, the full paths to their locations vary by cycle and + # cannot be set here (i.e. they can only be set in the loop over the + # cycles in the rocoto workflow XML file). + # + #----------------------------------------------------------------------- + # + global DATA_TABLE_FP, FIELD_TABLE_FP, FV3_NML_FN, FV3_NML_FP, NEMS_CONFIG_FP + DATA_TABLE_FP = os.path.join(EXPTDIR, DATA_TABLE_FN) + FIELD_TABLE_FP = os.path.join(EXPTDIR, FIELD_TABLE_FN) + FV3_NML_FN = os.path.splitext(FV3_NML_BASE_SUITE_FN)[0] + FV3_NML_FP = os.path.join(EXPTDIR, FV3_NML_FN) + NEMS_CONFIG_FP = os.path.join(EXPTDIR, NEMS_CONFIG_FN) + # + #----------------------------------------------------------------------- + # + # If USE_USER_STAGED_EXTRN_FILES is set to TRUE, make sure that the user- + # specified directories under which the external model files should be + # located actually exist. + # + #----------------------------------------------------------------------- + # + if USE_USER_STAGED_EXTRN_FILES: + + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS): + print_err_msg_exit(f''' + The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged + external model files for generating ICs should be located does not exist: + EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''') + + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS): + print_err_msg_exit(f''' + The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged + external model files for generating LBCs should be located does not exist: + EXTRN_MDL_SOURCE_BASEDIR_LBCS = \"{EXTRN_MDL_SOURCE_BASEDIR_LBCS}\"''') + # + #----------------------------------------------------------------------- + # + # Make sure that DO_ENSEMBLE is set to a valid value. Then set the names + # of the ensemble members. These will be used to set the ensemble member + # directories. Also, set the full path to the FV3 namelist file corresponding + # to each ensemble member. + # + #----------------------------------------------------------------------- + # + global NDIGITS_ENSMEM_NAMES,ENSMEM_NAMES,FV3_NML_ENSMEM_FPS,NUM_ENS_MEMBERS + NDIGITS_ENSMEM_NAMES=0 + ENSMEM_NAMES=[] + FV3_NML_ENSMEM_FPS=[] + if DO_ENSEMBLE: + NDIGITS_ENSMEM_NAMES=len(str(NUM_ENS_MEMBERS)) + fmt=f"0{NDIGITS_ENSMEM_NAMES}d" + for i in range(NUM_ENS_MEMBERS): + ENSMEM_NAMES.append(f"mem{fmt}".format(i+1)) + FV3_NML_ENSMEM_FPS.append(os.path.join(EXPTDIR, f"{FV3_NML_FN}_{ENSMEM_NAMES[i]}")) + # + #----------------------------------------------------------------------- + # + # Set the full path to the forecast model executable. + # + #----------------------------------------------------------------------- + # + global FV3_EXEC_FP + FV3_EXEC_FP = os.path.join(EXECDIR, FV3_EXEC_FN) + # + #----------------------------------------------------------------------- + # + # Set the full path to the script that can be used to (re)launch the + # workflow. Also, if USE_CRON_TO_RELAUNCH is set to TRUE, set the line + # to add to the cron table to automatically relaunch the workflow every + # CRON_RELAUNCH_INTVL_MNTS minutes. Otherwise, set the variable con- + # taining this line to a null string. + # + #----------------------------------------------------------------------- + # + global WFLOW_LAUNCH_SCRIPT_FP, WFLOW_LAUNCH_LOG_FP, CRONTAB_LINE + WFLOW_LAUNCH_SCRIPT_FP = os.path.join(USHDIR, WFLOW_LAUNCH_SCRIPT_FN) + WFLOW_LAUNCH_LOG_FP = os.path.join(EXPTDIR, WFLOW_LAUNCH_LOG_FN) + if USE_CRON_TO_RELAUNCH: + CRONTAB_LINE=f'''*/{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./{WFLOW_LAUNCH_SCRIPT_FN} called_from_cron="TRUE" >> ./{WFLOW_LAUNCH_LOG_FN} 2>&1''' + else: + CRONTAB_LINE="" + # + #----------------------------------------------------------------------- + # + # Set the full path to the script that, for a given task, loads the + # necessary module files and runs the tasks. + # + #----------------------------------------------------------------------- + # + global LOAD_MODULES_RUN_TASK_FP + LOAD_MODULES_RUN_TASK_FP = os.path.join(USHDIR, "load_modules_run_task.sh") + # + #----------------------------------------------------------------------- + # + # Define the various work subdirectories under the main work directory. + # Each of these corresponds to a different step/substep/task in the pre- + # processing, as follows: + # + # GRID_DIR: + # Directory in which the grid files will be placed (if RUN_TASK_MAKE_GRID + # is set to True) or searched for (if RUN_TASK_MAKE_GRID is set to + # False). + # + # OROG_DIR: + # Directory in which the orography files will be placed (if RUN_TASK_MAKE_OROG + # is set to True) or searched for (if RUN_TASK_MAKE_OROG is set to + # False). + # + # SFC_CLIMO_DIR: + # Directory in which the surface climatology files will be placed (if + # RUN_TASK_MAKE_SFC_CLIMO is set to True) or searched for (if + # RUN_TASK_MAKE_SFC_CLIMO is set to False). + # + #---------------------------------------------------------------------- + # + global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO + global GRID_DIR, OROG_DIR, SFC_CLIMO_DIR + global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID + + # + #----------------------------------------------------------------------- + # + # Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. + # + #----------------------------------------------------------------------- + # + if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): + print_err_msg_exit(f''' + Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"{DO_ENSEMBLE}\" + RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" + RUN_TASK_VX_ENSPOINT = \"{RUN_TASK_VX_ENSPOINT}\"''') + + if RUN_ENVIR == "nco": + + nco_fix_dir = os.path.join(FIXLAM_NCO_BASEDIR, PREDEF_GRID_NAME) + if not os.path.exists(nco_fix_dir): + print_err_msg_exit(f''' + The directory (nco_fix_dir) that should contain the pregenerated grid, + orography, and surface climatology files does not exist: + nco_fix_dir = \"{nco_fix_dir}\"''') + + if RUN_TASK_MAKE_GRID or \ + ( not RUN_TASK_MAKE_GRID and \ + GRID_DIR != nco_fix_dir ): + + msg=f''' + When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated + grid files already exist in the directory + + {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + + where + + FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" + + Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must + be set to \"FALSE\"), and the directory in which to look for the grid + files (i.e. GRID_DIR) must be set to the one above. Current values for + these quantities are: + + RUN_TASK_MAKE_GRID = \"{RUN_TASK_MAKE_GRID}\" + GRID_DIR = \"{GRID_DIR}\" + + Resetting RUN_TASK_MAKE_GRID to \"FALSE\" and GRID_DIR to the one above. + Reset values are: + ''' + + RUN_TASK_MAKE_GRID=False + GRID_DIR=nco_fix_dir + + msg+=f''' + RUN_TASK_MAKE_GRID = \"{RUN_TASK_MAKE_GRID}\" + GRID_DIR = \"{GRID_DIR}\" + ''' + + print_info_msg(msg) + + + if RUN_TASK_MAKE_OROG or \ + ( not RUN_TASK_MAKE_OROG and \ + OROG_DIR != nco_fix_dir ): + + msg=f''' + When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated + orography files already exist in the directory + {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + + where + + FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" + + Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must + be set to \"FALSE\"), and the directory in which to look for the orography + files (i.e. OROG_DIR) must be set to the one above. Current values for + these quantities are: + + RUN_TASK_MAKE_OROG = \"{RUN_TASK_MAKE_OROG}\" + OROG_DIR = \"{OROG_DIR}\" + + Resetting RUN_TASK_MAKE_OROG to \"FALSE\" and OROG_DIR to the one above. + Reset values are: + ''' + + RUN_TASK_MAKE_OROG=False + OROG_DIR=nco_fix_dir + + msg+=f''' + RUN_TASK_MAKE_OROG = \"{RUN_TASK_MAKE_OROG}\" + OROG_DIR = \"{OROG_DIR}\" + ''' + + print_info_msg(msg) + + + if RUN_TASK_MAKE_SFC_CLIMO or \ + ( not RUN_TASK_MAKE_SFC_CLIMO and \ + SFC_CLIMO_DIR != nco_fix_dir ): + + msg=f''' + When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated + surface climatology files already exist in the directory + + {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + + where + + FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" + + Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO + must be set to \"FALSE\"), and the directory in which to look for the + surface climatology files (i.e. SFC_CLIMO_DIR) must be set to the one + above. Current values for these quantities are: + + RUN_TASK_MAKE_SFC_CLIMO = \"{RUN_TASK_MAKE_SFC_CLIMO}\" + SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" + + Resetting RUN_TASK_MAKE_SFC_CLIMO to \"FALSE\" and SFC_CLIMO_DIR to the + one above. Reset values are: + ''' + + RUN_TASK_MAKE_SFC_CLIMO=False + SFC_CLIMO_DIR=nco_fix_dir + + msg+=f''' + RUN_TASK_MAKE_SFC_CLIMO = \"{RUN_TASK_MAKE_SFC_CLIMO}\" + SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" + ''' + + print_info_msg(msg) + + if RUN_TASK_VX_GRIDSTAT: + + msg=f''' + When RUN_ENVIR is set to \"nco\", it is assumed that the verification + will not be run. + RUN_TASK_VX_GRIDSTAT = \"{RUN_TASK_VX_GRIDSTAT}\" + Resetting RUN_TASK_VX_GRIDSTAT to \"FALSE\" + Reset value is:''' + + RUN_TASK_VX_GRIDSTAT=False + + msg+=f''' + RUN_TASK_VX_GRIDSTAT = \"{RUN_TASK_VX_GRIDSTAT}\" + ''' + + print_info_msg(msg) + + if RUN_TASK_VX_POINTSTAT: + + msg=f''' + When RUN_ENVIR is set to \"nco\", it is assumed that the verification + will not be run. + RUN_TASK_VX_POINTSTAT = \"{RUN_TASK_VX_POINTSTAT}\" + Resetting RUN_TASK_VX_POINTSTAT to \"FALSE\" + Reset value is:''' + + RUN_TASK_VX_POINTSTAT=False + + msg=f''' + RUN_TASK_VX_POINTSTAT = \"{RUN_TASK_VX_POINTSTAT}\" + ''' + + print_info_msg(msg) + + if RUN_TASK_VX_ENSGRID: + + msg=f''' + When RUN_ENVIR is set to \"nco\", it is assumed that the verification + will not be run. + RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" + Resetting RUN_TASK_VX_ENSGRID to \"FALSE\" + Reset value is:''' + + RUN_TASK_VX_ENSGRID=False + + msg+=f''' + RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" + ''' + + print_info_msg(msg) + + # + #----------------------------------------------------------------------- + # + # Now consider community mode. + # + #----------------------------------------------------------------------- + # + else: + # + # If RUN_TASK_MAKE_GRID is set to False, the workflow will look for + # the pregenerated grid files in GRID_DIR. In this case, make sure that + # GRID_DIR exists. Otherwise, set it to a predefined location under the + # experiment directory (EXPTDIR). + # + if not RUN_TASK_MAKE_GRID: + if not os.path.exists(GRID_DIR): + print_err_msg_exit(f''' + The directory (GRID_DIR) that should contain the pregenerated grid files + does not exist: + GRID_DIR = \"{GRID_DIR}\"''') + else: + GRID_DIR=os.path.join(EXPTDIR,"grid") + # + # If RUN_TASK_MAKE_OROG is set to False, the workflow will look for + # the pregenerated orography files in OROG_DIR. In this case, make sure + # that OROG_DIR exists. Otherwise, set it to a predefined location under + # the experiment directory (EXPTDIR). + # + if not RUN_TASK_MAKE_OROG: + if not os.path.exists(OROG_DIR): + print_err_msg_exit(f''' + The directory (OROG_DIR) that should contain the pregenerated orography + files does not exist: + OROG_DIR = \"{OROG_DIR}\"''') + else: + OROG_DIR=os.path.join(EXPTDIR,"orog") + # + # If RUN_TASK_MAKE_SFC_CLIMO is set to False, the workflow will look + # for the pregenerated surface climatology files in SFC_CLIMO_DIR. In + # this case, make sure that SFC_CLIMO_DIR exists. Otherwise, set it to + # a predefined location under the experiment directory (EXPTDIR). + # + if not RUN_TASK_MAKE_SFC_CLIMO: + if not os.path.exists(SFC_CLIMO_DIR): + print_err_msg_exit(f''' + The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface + climatology files does not exist: + SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''') + else: + SFC_CLIMO_DIR=os.path.join(EXPTDIR,"sfc_climo") + + #----------------------------------------------------------------------- + # + # Set cycle-independent parameters associated with the external models + # from which we will obtain the ICs and LBCs. + # + #----------------------------------------------------------------------- + # + + # export env vars before calling another module + export_vars() + + set_extrn_mdl_params() + + IMPORTS = ["EXTRN_MDL_SYSBASEDIR_ICS", "EXTRN_MDL_SYSBASEDIR_LBCS", "EXTRN_MDL_LBCS_OFFSET_HRS"] + import_vars(env_vars=IMPORTS) + # + #----------------------------------------------------------------------- + # + # Any regional model must be supplied lateral boundary conditions (in + # addition to initial conditions) to be able to perform a forecast. In + # the FV3-LAM model, these boundary conditions (BCs) are supplied using a + # "halo" of grid cells around the regional domain that extend beyond the + # boundary of the domain. The model is formulated such that along with + # files containing these BCs, it needs as input the following files (in + # NetCDF format): + # + # 1) A grid file that includes a halo of 3 cells beyond the boundary of + # the domain. + # 2) A grid file that includes a halo of 4 cells beyond the boundary of + # the domain. + # 3) A (filtered) orography file without a halo, i.e. a halo of width + # 0 cells. + # 4) A (filtered) orography file that includes a halo of 4 cells beyond + # the boundary of the domain. + # + # Note that the regional grid is referred to as "tile 7" in the code. + # We will let: + # + # * NH0 denote the width (in units of number of cells on tile 7) of + # the 0-cell-wide halo, i.e. NH0 = 0; + # + # * NH3 denote the width (in units of number of cells on tile 7) of + # the 3-cell-wide halo, i.e. NH3 = 3; and + # + # * NH4 denote the width (in units of number of cells on tile 7) of + # the 4-cell-wide halo, i.e. NH4 = 4. + # + # We define these variables next. + # + #----------------------------------------------------------------------- + # + global NH0,NH3,NH4 + NH0=0 + NH3=3 + NH4=4 + + # export env vars + EXPORTS = ["NH0","NH3","NH4"] + export_vars(env_vars = EXPORTS) + # + #----------------------------------------------------------------------- + # + # Set parameters according to the type of horizontal grid generation + # method specified. First consider GFDL's global-parent-grid based + # method. + # + #----------------------------------------------------------------------- + # + global LON_CTR,LAT_CTR,NX,NY,NHW,STRETCH_FAC,\ + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG,\ + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG,\ + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG,\ + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG + global PAZI,DEL_ANGLE_X_SG,DEL_ANGLE_Y_SG,\ + NEG_NX_OF_DOM_WITH_WIDE_HALO,\ + NEG_NY_OF_DOM_WITH_WIDE_HALO + + if GRID_GEN_METHOD == "GFDLgrid": + + (\ + LON_CTR,LAT_CTR,NX,NY,NHW,STRETCH_FAC, + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG \ + ) = \ + set_gridparams_GFDLgrid( \ + lon_of_t6_ctr=GFDLgrid_LON_T6_CTR, \ + lat_of_t6_ctr=GFDLgrid_LAT_T6_CTR, \ + res_of_t6g=GFDLgrid_RES, \ + stretch_factor=GFDLgrid_STRETCH_FAC, \ + refine_ratio_t6g_to_t7g=GFDLgrid_REFINE_RATIO, \ + istart_of_t7_on_t6g=GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G, \ + iend_of_t7_on_t6g=GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G, \ + jstart_of_t7_on_t6g=GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G, \ + jend_of_t7_on_t6g=GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G) + # + #----------------------------------------------------------------------- + # + # Now consider Jim Purser's map projection/grid generation method. + # + #----------------------------------------------------------------------- + # + elif GRID_GEN_METHOD == "ESGgrid": + + (\ + LON_CTR,LAT_CTR,NX,NY,PAZI, + NHW,STRETCH_FAC,DEL_ANGLE_X_SG,DEL_ANGLE_Y_SG, + NEG_NX_OF_DOM_WITH_WIDE_HALO, + NEG_NY_OF_DOM_WITH_WIDE_HALO \ + ) = \ + set_gridparams_ESGgrid( \ + lon_ctr=ESGgrid_LON_CTR, \ + lat_ctr=ESGgrid_LAT_CTR, \ + nx=ESGgrid_NX, \ + ny=ESGgrid_NY, \ + pazi=ESGgrid_PAZI, \ + halo_width=ESGgrid_WIDE_HALO_WIDTH, \ + delx=ESGgrid_DELX, \ + dely=ESGgrid_DELY) + + # + #----------------------------------------------------------------------- + # + # Create a new experiment directory. Note that at this point we are + # guaranteed that there is no preexisting experiment directory. For + # platforms with no workflow manager, we need to create LOGDIR as well, + # since it won't be created later at runtime. + # + #----------------------------------------------------------------------- + # + mkdir_vrfy(f' -p "{EXPTDIR}"') + mkdir_vrfy(f' -p "{LOGDIR}"') + # + #----------------------------------------------------------------------- + # + # If not running the MAKE_GRID_TN, MAKE_OROG_TN, and/or MAKE_SFC_CLIMO + # tasks, create symlinks under the FIXLAM directory to pregenerated grid, + # orography, and surface climatology files. In the process, also set + # RES_IN_FIXLAM_FILENAMES, which is the resolution of the grid (in units + # of number of grid points on an equivalent global uniform cubed-sphere + # grid) used in the names of the fixed files in the FIXLAM directory. + # + #----------------------------------------------------------------------- + # + mkdir_vrfy(f' -p "{FIXLAM}"') + RES_IN_FIXLAM_FILENAMES="" + # + #----------------------------------------------------------------------- + # + # If the grid file generation task in the workflow is going to be skipped + # (because pregenerated files are available), create links in the FIXLAM + # directory to the pregenerated grid files. + # + #----------------------------------------------------------------------- + # + + # export env vars + export_vars() + + # link fix files + res_in_grid_fns="" + if not RUN_TASK_MAKE_GRID: + + res_in_grid_fns = link_fix( \ + verbose=VERBOSE, \ + file_group="grid") + + RES_IN_FIXLAM_FILENAMES=res_in_grid_fns + # + #----------------------------------------------------------------------- + # + # If the orography file generation task in the workflow is going to be + # skipped (because pregenerated files are available), create links in + # the FIXLAM directory to the pregenerated orography files. + # + #----------------------------------------------------------------------- + # + res_in_orog_fns="" + if not RUN_TASK_MAKE_OROG: + + res_in_orog_fns = link_fix( \ + verbose=VERBOSE, \ + file_group="orog") + + if not RES_IN_FIXLAM_FILENAMES and \ + ( res_in_orog_fns != RES_IN_FIXLAM_FILENAMES): + print_err_msg_exit(f''' + The resolution extracted from the orography file names (res_in_orog_fns) + does not match the resolution in other groups of files already consi- + dered (RES_IN_FIXLAM_FILENAMES): + res_in_orog_fns = {res_in_orog_fns} + RES_IN_FIXLAM_FILENAMES = {RES_IN_FIXLAM_FILENAMES}''') + else: + RES_IN_FIXLAM_FILENAMES=res_in_orog_fns + # + #----------------------------------------------------------------------- + # + # If the surface climatology file generation task in the workflow is + # going to be skipped (because pregenerated files are available), create + # links in the FIXLAM directory to the pregenerated surface climatology + # files. + # + #----------------------------------------------------------------------- + # + res_in_sfc_climo_fns="" + if not RUN_TASK_MAKE_SFC_CLIMO: + + res_in_sfc_climo_fns = link_fix( \ + verbose=VERBOSE, \ + file_group="sfc_climo") + + if RES_IN_FIXLAM_FILENAMES and \ + res_in_sfc_climo_fns != RES_IN_FIXLAM_FILENAMES: + print_err_msg_exit(f''' + The resolution extracted from the surface climatology file names (res_- + in_sfc_climo_fns) does not match the resolution in other groups of files + already considered (RES_IN_FIXLAM_FILENAMES): + res_in_sfc_climo_fns = {res_in_sfc_climo_fns} + RES_IN_FIXLAM_FILENAMES = {RES_IN_FIXLAM_FILENAMES}''') + else: + RES_IN_FIXLAM_FILENAMES=res_in_sfc_climo_fns + # + #----------------------------------------------------------------------- + # + # The variable CRES is needed in constructing various file names. If + # not running the make_grid task, we can set it here. Otherwise, it + # will get set to a valid value by that task. + # + #----------------------------------------------------------------------- + # + global CRES + CRES="" + if not RUN_TASK_MAKE_GRID: + CRES=f"C{RES_IN_FIXLAM_FILENAMES}" + # + #----------------------------------------------------------------------- + # + # Make sure that WRITE_DOPOST is set to a valid value. + # + #----------------------------------------------------------------------- + # + global RUN_TASK_RUN_POST + if WRITE_DOPOST: + # Turn off run_post + RUN_TASK_RUN_POST=False + + # Check if SUB_HOURLY_POST is on + if SUB_HOURLY_POST: + print_err_msg_exit(f''' + SUB_HOURLY_POST is NOT available with Inline Post yet.''') + # + #----------------------------------------------------------------------- + # + # Calculate PE_MEMBER01. This is the number of MPI tasks used for the + # forecast, including those for the write component if QUILTING is set + # to True. + # + #----------------------------------------------------------------------- + # + global PE_MEMBER01 + PE_MEMBER01=LAYOUT_X*LAYOUT_Y + if QUILTING: + PE_MEMBER01 = PE_MEMBER01 + WRTCMP_write_groups*WRTCMP_write_tasks_per_group + + print_info_msg(f''' + The number of MPI tasks for the forecast (including those for the write + component if it is being used) are: + PE_MEMBER01 = {PE_MEMBER01}''', verbose=VERBOSE) + # + #----------------------------------------------------------------------- + # + # Calculate the number of nodes (NNODES_RUN_FCST) to request from the job + # scheduler for the forecast task (RUN_FCST_TN). This is just PE_MEMBER01 + # dividied by the number of processes per node we want to request for this + # task (PPN_RUN_FCST), then rounded up to the nearest integer, i.e. + # + # NNODES_RUN_FCST = ceil(PE_MEMBER01/PPN_RUN_FCST) + # + # where ceil(...) is the ceiling function, i.e. it rounds its floating + # point argument up to the next larger integer. Since in bash, division + # of two integers returns a truncated integer, and since bash has no + # built-in ceil(...) function, we perform the rounding-up operation by + # adding the denominator (of the argument of ceil(...) above) minus 1 to + # the original numerator, i.e. by redefining NNODES_RUN_FCST to be + # + # NNODES_RUN_FCST = (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST + # + #----------------------------------------------------------------------- + # + global NNODES_RUN_FCST + NNODES_RUN_FCST= (PE_MEMBER01 + PPN_RUN_FCST - 1)//PPN_RUN_FCST + + # + #----------------------------------------------------------------------- + # + # Call the function that checks whether the RUC land surface model (LSM) + # is being called by the physics suite and sets the workflow variable + # SDF_USES_RUC_LSM to True or False accordingly. + # + #----------------------------------------------------------------------- + # + global SDF_USES_RUC_LSM + SDF_USES_RUC_LSM = check_ruc_lsm( \ + ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP) + # + #----------------------------------------------------------------------- + # + # Set the name of the file containing aerosol climatology data that, if + # necessary, can be used to generate approximate versions of the aerosol + # fields needed by Thompson microphysics. This file will be used to + # generate such approximate aerosol fields in the ICs and LBCs if Thompson + # MP is included in the physics suite and if the exteranl model for ICs + # or LBCs does not already provide these fields. Also, set the full path + # to this file. + # + #----------------------------------------------------------------------- + # + THOMPSON_MP_CLIMO_FN="Thompson_MP_MONTHLY_CLIMO.nc" + THOMPSON_MP_CLIMO_FP=os.path.join(FIXam,THOMPSON_MP_CLIMO_FN) + # + #----------------------------------------------------------------------- + # + # Call the function that, if the Thompson microphysics parameterization + # is being called by the physics suite, modifies certain workflow arrays + # to ensure that fixed files needed by this parameterization are copied + # to the FIXam directory and appropriate symlinks to them are created in + # the run directories. This function also sets the workflow variable + # SDF_USES_THOMPSON_MP that indicates whether Thompson MP is called by + # the physics suite. + # + #----------------------------------------------------------------------- + # + SDF_USES_THOMPSON_MP = set_thompson_mp_fix_files( \ + ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP, \ + thompson_mp_climo_fn=THOMPSON_MP_CLIMO_FN) + + IMPORTS = [ "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam" ] + import_vars(env_vars=IMPORTS) + # + #----------------------------------------------------------------------- + # + # Generate the shell script that will appear in the experiment directory + # (EXPTDIR) and will contain definitions of variables needed by the va- + # rious scripts in the workflow. We refer to this as the experiment/ + # workflow global variable definitions file. We will create this file + # by: + # + # 1) Copying the default workflow/experiment configuration file (speci- + # fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- + # rectory specified by USHDIR) to the experiment directory and rena- + # ming it to the name specified by GLOBAL_VAR_DEFNS_FN. + # + # 2) Resetting the default variable values in this file to their current + # values. This is necessary because these variables may have been + # reset by the user-specified configuration file (if one exists in + # USHDIR) and/or by this setup script, e.g. because predef_domain is + # set to a valid non-empty value. + # + # 3) Appending to the variable definitions file any new variables intro- + # duced in this setup script that may be needed by the scripts that + # perform the various tasks in the workflow (and which source the va- + # riable defintions file). + # + # First, set the full path to the variable definitions file and copy the + # default configuration script into it. + # + #----------------------------------------------------------------------- + # + + # update dictionary with globals() values + update_dict = {k: globals()[k] for k in cfg_d.keys() if k in globals() } + cfg_d.update(update_dict) + + # write the updated default dictionary + global GLOBAL_VAR_DEFNS_FP + GLOBAL_VAR_DEFNS_FP=os.path.join(EXPTDIR,GLOBAL_VAR_DEFNS_FN) + all_lines=cfg_to_shell_str(cfg_d) + with open(GLOBAL_VAR_DEFNS_FP,'w') as f: + msg = f""" # + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # Section 1: + # This section contains (most of) the primary experiment variables, i.e. + # those variables that are defined in the default configuration file + # (config_defaults.sh) and that can be reset via the user-specified + # experiment configuration file (config.sh). + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # + """ + f.write(dedent(msg)) + f.write(all_lines) + + # print info message + msg=dedent(f''' + Before updating default values of experiment variables to user-specified + values, the variable \"line_list\" contains: + + ''') + + msg +=dedent(f''' + {all_lines}''') + + print_info_msg(msg,verbose=DEBUG) + # + # print info message + # + print_info_msg(f''' + Generating the global experiment variable definitions file specified by + GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"{GLOBAL_VAR_DEFNS_FN}\" + Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"{GLOBAL_VAR_DEFNS_FP}\" + For more detailed information, set DEBUG to \"TRUE\" in the experiment + configuration file (\"{EXPT_CONFIG_FN}\").''') + + # + #----------------------------------------------------------------------- + # + # Append additional variable definitions (and comments) to the variable + # definitions file. These variables have been set above using the vari- + # ables in the default and local configuration scripts. These variables + # are needed by various tasks/scripts in the workflow. + # + #----------------------------------------------------------------------- + # + msg = f""" + # + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # Section 2: + # This section defines variables that have been derived from the primary + # set of experiment variables above (we refer to these as \"derived\" or + # \"secondary\" variables). + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # + + # + #----------------------------------------------------------------------- + # + # Full path to workflow (re)launch script, its log file, and the line + # that gets added to the cron table to launch this script if the flag + # USE_CRON_TO_RELAUNCH is set to \"TRUE\". + # + #----------------------------------------------------------------------- + # + WFLOW_LAUNCH_SCRIPT_FP='{WFLOW_LAUNCH_SCRIPT_FP}' + WFLOW_LAUNCH_LOG_FP='{WFLOW_LAUNCH_LOG_FP}' + CRONTAB_LINE='{CRONTAB_LINE}' + # + #----------------------------------------------------------------------- + # + # Directories. + # + #----------------------------------------------------------------------- + # + SR_WX_APP_TOP_DIR='{SR_WX_APP_TOP_DIR}' + HOMErrfs='{HOMErrfs}' + USHDIR='{USHDIR}' + SCRIPTSDIR='{SCRIPTSDIR}' + JOBSDIR='{JOBSDIR}' + SORCDIR='{SORCDIR}' + SRC_DIR='{SRC_DIR}' + PARMDIR='{PARMDIR}' + MODULES_DIR='{MODULES_DIR}' + EXECDIR='{EXECDIR}' + FIXam='{FIXam}' + FIXclim='{FIXclim}' + FIXLAM='{FIXLAM}' + FIXgsm='{FIXgsm}' + FIXaer='{FIXaer}' + FIXlut='{FIXlut}' + COMROOT='{COMROOT}' + COMOUT_BASEDIR='{COMOUT_BASEDIR}' + TEMPLATE_DIR='{TEMPLATE_DIR}' + VX_CONFIG_DIR='{VX_CONFIG_DIR}' + METPLUS_CONF='{METPLUS_CONF}' + MET_CONFIG='{MET_CONFIG}' + UFS_WTHR_MDL_DIR='{UFS_WTHR_MDL_DIR}' + UFS_UTILS_DIR='{UFS_UTILS_DIR}' + SFC_CLIMO_INPUT_DIR='{SFC_CLIMO_INPUT_DIR}' + TOPO_DIR='{TOPO_DIR}' + UPP_DIR='{UPP_DIR}' + + EXPTDIR='{EXPTDIR}' + LOGDIR='{LOGDIR}' + CYCLE_BASEDIR='{CYCLE_BASEDIR}' + GRID_DIR='{GRID_DIR}' + OROG_DIR='{OROG_DIR}' + SFC_CLIMO_DIR='{SFC_CLIMO_DIR}' + + NDIGITS_ENSMEM_NAMES='{NDIGITS_ENSMEM_NAMES}' + ENSMEM_NAMES={list_to_str(ENSMEM_NAMES)} + FV3_NML_ENSMEM_FPS={list_to_str(FV3_NML_ENSMEM_FPS)} + # + #----------------------------------------------------------------------- + # + # Files. + # + #----------------------------------------------------------------------- + # + GLOBAL_VAR_DEFNS_FP='{GLOBAL_VAR_DEFNS_FP}' + + DATA_TABLE_FN='{DATA_TABLE_FN}' + DIAG_TABLE_FN='{DIAG_TABLE_FN}' + FIELD_TABLE_FN='{FIELD_TABLE_FN}' + MODEL_CONFIG_FN='{MODEL_CONFIG_FN}' + NEMS_CONFIG_FN='{NEMS_CONFIG_FN}' + + DATA_TABLE_TMPL_FN='{DATA_TABLE_TMPL_FN}' + DIAG_TABLE_TMPL_FN='{DIAG_TABLE_TMPL_FN}' + FIELD_TABLE_TMPL_FN='{FIELD_TABLE_TMPL_FN}' + MODEL_CONFIG_TMPL_FN='{MODEL_CONFIG_TMPL_FN}' + NEMS_CONFIG_TMPL_FN='{NEMS_CONFIG_TMPL_FN}' + + DATA_TABLE_TMPL_FP='{DATA_TABLE_TMPL_FP}' + DIAG_TABLE_TMPL_FP='{DIAG_TABLE_TMPL_FP}' + FIELD_TABLE_TMPL_FP='{FIELD_TABLE_TMPL_FP}' + FV3_NML_BASE_SUITE_FP='{FV3_NML_BASE_SUITE_FP}' + FV3_NML_YAML_CONFIG_FP='{FV3_NML_YAML_CONFIG_FP}' + FV3_NML_BASE_ENS_FP='{FV3_NML_BASE_ENS_FP}' + MODEL_CONFIG_TMPL_FP='{MODEL_CONFIG_TMPL_FP}' + NEMS_CONFIG_TMPL_FP='{NEMS_CONFIG_TMPL_FP}' + + CCPP_PHYS_SUITE_FN='{CCPP_PHYS_SUITE_FN}' + CCPP_PHYS_SUITE_IN_CCPP_FP='{CCPP_PHYS_SUITE_IN_CCPP_FP}' + CCPP_PHYS_SUITE_FP='{CCPP_PHYS_SUITE_FP}' + + FIELD_DICT_FN='{FIELD_DICT_FN}' + FIELD_DICT_IN_UWM_FP='{FIELD_DICT_IN_UWM_FP}' + FIELD_DICT_FP='{FIELD_DICT_FP}' + + DATA_TABLE_FP='{DATA_TABLE_FP}' + FIELD_TABLE_FP='{FIELD_TABLE_FP}' + FV3_NML_FN='{FV3_NML_FN}' # This may not be necessary... + FV3_NML_FP='{FV3_NML_FP}' + NEMS_CONFIG_FP='{NEMS_CONFIG_FP}' + + FV3_EXEC_FP='{FV3_EXEC_FP}' + + LOAD_MODULES_RUN_TASK_FP='{LOAD_MODULES_RUN_TASK_FP}' + + THOMPSON_MP_CLIMO_FN='{THOMPSON_MP_CLIMO_FN}' + THOMPSON_MP_CLIMO_FP='{THOMPSON_MP_CLIMO_FP}' + # + #----------------------------------------------------------------------- + # + # Flag for creating relative symlinks (as opposed to absolute ones). + # + #----------------------------------------------------------------------- + # + RELATIVE_LINK_FLAG='{RELATIVE_LINK_FLAG}' + # + #----------------------------------------------------------------------- + # + # Parameters that indicate whether or not various parameterizations are + # included in and called by the physics suite. + # + #----------------------------------------------------------------------- + # + SDF_USES_RUC_LSM='{type_to_str(SDF_USES_RUC_LSM)}' + SDF_USES_THOMPSON_MP='{type_to_str(SDF_USES_THOMPSON_MP)}' + # + #----------------------------------------------------------------------- + # + # Grid configuration parameters needed regardless of grid generation + # method used. + # + #----------------------------------------------------------------------- + # + GTYPE='{GTYPE}' + TILE_RGNL='{TILE_RGNL}' + NH0='{NH0}' + NH3='{NH3}' + NH4='{NH4}' + + LON_CTR='{LON_CTR}' + LAT_CTR='{LAT_CTR}' + NX='{NX}' + NY='{NY}' + NHW='{NHW}' + STRETCH_FAC='{STRETCH_FAC}' + + RES_IN_FIXLAM_FILENAMES='{RES_IN_FIXLAM_FILENAMES}' + # + # If running the make_grid task, CRES will be set to a null string during + # the grid generation step. It will later be set to an actual value after + # the make_grid task is complete. + # + CRES='{CRES}'""" + with open(GLOBAL_VAR_DEFNS_FP,'a') as f: + f.write(dedent(msg)) + # + #----------------------------------------------------------------------- + # + # Append to the variable definitions file the defintions of grid parame- + # ters that are specific to the grid generation method used. + # + #----------------------------------------------------------------------- + # + if GRID_GEN_METHOD == "GFDLgrid": + + msg=f""" + # + #----------------------------------------------------------------------- + # + # Grid configuration parameters for a regional grid generated from a + # global parent cubed-sphere grid. This is the method originally + # suggested by GFDL since it allows GFDL's nested grid generator to be + # used to generate a regional grid. However, for large regional domains, + # it results in grids that have an unacceptably large range of cell sizes + # (i.e. ratio of maximum to minimum cell size is not sufficiently close + # to 1). + # + #----------------------------------------------------------------------- + # + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}'""" + with open(GLOBAL_VAR_DEFNS_FP,'a') as f: + f.write(dedent(msg)) + + elif GRID_GEN_METHOD == "ESGgrid": + + msg=f""" + # + #----------------------------------------------------------------------- + # + # Grid configuration parameters for a regional grid generated independently + # of a global parent grid. This method was developed by Jim Purser of + # EMC and results in very uniform grids (i.e. ratio of maximum to minimum + # cell size is very close to 1). + # + #----------------------------------------------------------------------- + # + DEL_ANGLE_X_SG='{DEL_ANGLE_X_SG}' + DEL_ANGLE_Y_SG='{DEL_ANGLE_Y_SG}' + NEG_NX_OF_DOM_WITH_WIDE_HALO='{NEG_NX_OF_DOM_WITH_WIDE_HALO}' + NEG_NY_OF_DOM_WITH_WIDE_HALO='{NEG_NY_OF_DOM_WITH_WIDE_HALO}' + PAZI='{PAZI or ''}'""" + with open(GLOBAL_VAR_DEFNS_FP,'a') as f: + f.write(dedent(msg)) + # + #----------------------------------------------------------------------- + # + # Continue appending variable definitions to the variable definitions + # file. + # + #----------------------------------------------------------------------- + # + msg = f""" + # + #----------------------------------------------------------------------- + # + # Flag in the \"{MODEL_CONFIG_FN}\" file for coupling the ocean model to + # the weather model. + # + #----------------------------------------------------------------------- + # + CPL='{type_to_str(CPL)}' + # + #----------------------------------------------------------------------- + # + # Name of the ozone parameterization. The value this gets set to depends + # on the CCPP physics suite being used. + # + #----------------------------------------------------------------------- + # + OZONE_PARAM='{OZONE_PARAM}' + # + #----------------------------------------------------------------------- + # + # If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system + # directory in which the workflow scripts will look for the files generated + # by the external model specified in EXTRN_MDL_NAME_ICS. These files will + # be used to generate the input initial condition and surface files for + # the FV3-LAM. + # + #----------------------------------------------------------------------- + # + EXTRN_MDL_SYSBASEDIR_ICS='{EXTRN_MDL_SYSBASEDIR_ICS}' + # + #----------------------------------------------------------------------- + # + # If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system + # directory in which the workflow scripts will look for the files generated + # by the external model specified in EXTRN_MDL_NAME_LBCS. These files + # will be used to generate the input lateral boundary condition files for + # the FV3-LAM. + # + #----------------------------------------------------------------------- + # + EXTRN_MDL_SYSBASEDIR_LBCS='{EXTRN_MDL_SYSBASEDIR_LBCS}' + # + #----------------------------------------------------------------------- + # + # Shift back in time (in units of hours) of the starting time of the ex- + # ternal model specified in EXTRN_MDL_NAME_LBCS. + # + #----------------------------------------------------------------------- + # + EXTRN_MDL_LBCS_OFFSET_HRS='{EXTRN_MDL_LBCS_OFFSET_HRS}' + # + #----------------------------------------------------------------------- + # + # Boundary condition update times (in units of forecast hours). Note that + # LBC_SPEC_FCST_HRS is an array, even if it has only one element. + # + #----------------------------------------------------------------------- + # + LBC_SPEC_FCST_HRS={list_to_str(LBC_SPEC_FCST_HRS)} + # + #----------------------------------------------------------------------- + # + # The number of cycles for which to make forecasts and the list of + # starting dates/hours of these cycles. + # + #----------------------------------------------------------------------- + # + NUM_CYCLES='{NUM_CYCLES}' + ALL_CDATES={list_to_str(ALL_CDATES)} + # + #----------------------------------------------------------------------- + # + # Parameters that determine whether FVCOM data will be used, and if so, + # their location. + # + # If USE_FVCOM is set to \"TRUE\", then FVCOM data (in the file FVCOM_FILE + # located in the directory FVCOM_DIR) will be used to update the surface + # boundary conditions during the initial conditions generation task + # (MAKE_ICS_TN). + # + #----------------------------------------------------------------------- + # + USE_FVCOM='{type_to_str(USE_FVCOM)}' + FVCOM_DIR='{FVCOM_DIR}' + FVCOM_FILE='{FVCOM_FILE}' + # + #----------------------------------------------------------------------- + # + # Computational parameters. + # + #----------------------------------------------------------------------- + # + NCORES_PER_NODE='{NCORES_PER_NODE}' + PE_MEMBER01='{PE_MEMBER01}' + # + #----------------------------------------------------------------------- + # + # IF DO_SPP is set to "TRUE", N_VAR_SPP specifies the number of physics + # parameterizations that are perturbed with SPP. If DO_LSM_SPP is set to + # "TRUE", N_VAR_LNDP specifies the number of LSM parameters that are + # perturbed. LNDP_TYPE determines the way LSM perturbations are employed + # and FHCYC_LSM_SPP_OR_NOT sets FHCYC based on whether LSM perturbations + # are turned on or not. + # + #----------------------------------------------------------------------- + # + N_VAR_SPP='{N_VAR_SPP}' + N_VAR_LNDP='{N_VAR_LNDP}' + LNDP_TYPE='{LNDP_TYPE}' + FHCYC_LSM_SPP_OR_NOT='{FHCYC_LSM_SPP_OR_NOT}' + """ + + with open(GLOBAL_VAR_DEFNS_FP,'a') as f: + f.write(dedent(msg)) + + # export all vars + export_vars() + + # + #----------------------------------------------------------------------- + # + # Check validity of parameters in one place, here in the end. + # + #----------------------------------------------------------------------- + # + + # update dictionary with globals() values + update_dict = {k: globals()[k] for k in cfg_d.keys() if k in globals() } + cfg_d.update(update_dict) + + # loop through cfg_d and check validity of params + cfg_v = load_config_file("valid_param_vals.yaml") + for k,v in cfg_d.items(): + if v == None: + continue + vkey = 'valid_vals_' + k + if (vkey in cfg_v) and not (v in cfg_v[vkey]): + print_err_msg_exit(f''' + The variable {k}={v} in {EXPT_DEFAULT_CONFIG_FN} or {EXPT_CONFIG_FN} does not have + a valid value. Possible values are: + {k} = {cfg_v[vkey]}''') + + # + #----------------------------------------------------------------------- + # + # Print message indicating successful completion of script. + # + #----------------------------------------------------------------------- + # + print_info_msg(f''' + ======================================================================== + Function setup() in \"{os.path.basename(__file__)}\" completed successfully!!! + ========================================================================''') + +# +#----------------------------------------------------------------------- +# +# Call the function defined above. +# +#----------------------------------------------------------------------- +# +if __name__ == "__main__": + setup() + diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.facsf.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.maximum_snow_albedo.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.slope_type.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.snowfree_albedo.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.soil_type.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.substrate_temperature.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_greenness.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357.vegetation_type.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo3.nc b/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo3.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo6.nc b/ush/test_data/RRFS_CONUS_3km/C3357_grid.tile7.halo6.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo3.nc b/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo3.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo6.nc b/ush/test_data/RRFS_CONUS_3km/C3357_mosaic.halo6.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo4.nc b/ush/test_data/RRFS_CONUS_3km/C3357_oro_data.tile7.halo4.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ls.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ls.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ss.tile7.halo0.nc b/ush/test_data/RRFS_CONUS_3km/C3357_oro_data_ss.tile7.halo0.nc new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/test_data/suite_FV3_GSD_SAR.xml b/ush/test_data/suite_FV3_GSD_SAR.xml new file mode 100644 index 0000000000..f2b38d5773 --- /dev/null +++ b/ush/test_data/suite_FV3_GSD_SAR.xml @@ -0,0 +1,85 @@ + + + + + + + GFS_time_vary_pre + GFS_rrtmg_setup + GFS_rad_time_vary + GFS_phys_time_vary + + + + + GFS_suite_interstitial_rad_reset + sgscloud_radpre + GFS_rrtmg_pre + GFS_radiation_surface + rrtmg_sw_pre + rrtmg_sw + rrtmg_sw_post + rrtmg_lw_pre + rrtmg_lw + sgscloud_radpost + rrtmg_lw_post + GFS_rrtmg_post + + + + + GFS_suite_interstitial_phys_reset + GFS_suite_stateout_reset + get_prs_fv3 + GFS_suite_interstitial_1 + GFS_surface_generic_pre + GFS_surface_composites_pre + dcyc2t3 + GFS_surface_composites_inter + GFS_suite_interstitial_2 + + + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst_pre + sfc_nst + sfc_nst_post + lsm_ruc + GFS_surface_loop_control_part2 + + + + GFS_surface_composites_post + sfc_diag + sfc_diag_post + GFS_surface_generic_post + mynnedmf_wrapper + GFS_GWD_generic_pre + cires_ugwp + cires_ugwp_post + GFS_GWD_generic_post + GFS_suite_stateout_update + ozphys_2015 + h2ophys + get_phi_fv3 + + GFS_suite_interstitial_3 + GFS_suite_interstitial_4 + + GFS_MP_generic_pre + mp_thompson_pre + mp_thompson + mp_thompson_post + GFS_MP_generic_post + maximum_hourly_diagnostics + phys_tend + + + + + GFS_stochastics + + + + diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml new file mode 100644 index 0000000000..f9a00e95b4 --- /dev/null +++ b/ush/valid_param_vals.yaml @@ -0,0 +1,84 @@ +# +# Define valid values for various global experiment/workflow variables. +# +valid_vals_RUN_ENVIR: ["nco", "community"] +valid_vals_VERBOSE: [True, False] +valid_vals_DEBUG: [True, False] +valid_vals_MACHINE: ["WCOSS_DELL_P3", "HERA", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] +valid_vals_FCST_MODEL: ["ufs-weather-model", "fv3gfs_aqm"] +valid_vals_WORKFLOW_MANAGER: ["rocoto", "none"] +valid_vals_PREDEF_GRID_NAME: [ +"RRFS_CONUS_25km", +"RRFS_CONUS_13km", +"RRFS_CONUS_3km", +"RRFS_CONUScompact_25km", +"RRFS_CONUScompact_13km", +"RRFS_CONUScompact_3km", +"RRFS_SUBCONUS_3km", +"RRFS_AK_13km", +"RRFS_AK_3km", +"CONUS_25km_GFDLgrid", +"CONUS_3km_GFDLgrid", +"EMC_AK", +"EMC_HI", +"EMC_PR", +"EMC_GU", +"GSL_HAFSV0.A_25km", +"GSL_HAFSV0.A_13km", +"GSL_HAFSV0.A_3km", +"GSD_HRRR_AK_50km", +"RRFS_NA_13km", +"RRFS_NA_3km", +"SUBCONUS_Ind_3km" +] +valid_vals_CCPP_PHYS_SUITE: [ +"FV3_GFS_2017_gfdlmp", +"FV3_GFS_2017_gfdlmp_regional", +"FV3_GFS_v15p2", +"FV3_GFS_v15_thompson_mynn_lam3km", +"FV3_GFS_v16", +"FV3_RRFS_v1beta", +"FV3_RRFS_v1alpha", +"FV3_HRRR" +] +valid_vals_GFDLgrid_RES: [48, 96, 192, 384, 768, 1152, 3072] +valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "RAP", "HRRR", "NAM"] +valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "RAP", "HRRR", "NAM"] +valid_vals_USE_USER_STAGED_EXTRN_FILES: [True, False] +valid_vals_FV3GFS_FILE_FMT_ICS: ["nemsio", "grib2", "netcdf"] +valid_vals_FV3GFS_FILE_FMT_LBCS: ["nemsio", "grib2", "netcdf"] +valid_vals_GRID_GEN_METHOD: ["GFDLgrid", "ESGgrid"] +valid_vals_PREEXISTING_DIR_METHOD: ["delete", "rename", "quit"] +valid_vals_GTYPE: ["regional"] +valid_vals_WRTCMP_output_grid: ["rotated_latlon", "lambert_conformal", "regional_latlon"] +valid_vals_RUN_TASK_MAKE_GRID: [True, False] +valid_vals_RUN_TASK_MAKE_OROG: [True, False] +valid_vals_RUN_TASK_MAKE_SFC_CLIMO: [True, False] +valid_vals_RUN_TASK_RUN_POST: [True, False] +valid_vals_WRITE_DOPOST: [True, False] +valid_vals_RUN_TASK_VX_GRIDSTAT: [True, False] +valid_vals_RUN_TASK_VX_POINTSTAT: [True, False] +valid_vals_RUN_TASK_VX_ENSGRID: [True, False] +valid_vals_RUN_TASK_VX_ENSPOINT: [True, False] +valid_vals_QUILTING: [True, False] +valid_vals_PRINT_ESMF: [True, False] +valid_vals_USE_CRON_TO_RELAUNCH: [True, False] +valid_vals_DOT_OR_USCORE: [".", "_"] +valid_vals_NOMADS: [True, False] +valid_vals_NOMADS_file_type: ["GRIB2", "grib2", "NEMSIO", "nemsio"] +valid_vals_DO_ENSEMBLE: [True, False] +valid_vals_USE_CUSTOM_POST_CONFIG_FILE: [True, False] +valid_vals_USE_CRTM: [True, False] +valid_vals_DO_SHUM: [True, False] +valid_vals_DO_SPPT: [True, False] +valid_vals_DO_SPP: [True, False] +valid_vals_DO_LSM_SPP: [True, False] +valid_vals_DO_SKEB: [True, False] +valid_vals_USE_ZMTNBLCK: [True, False] +valid_vals_USE_FVCOM: [True, False] +valid_vals_FVCOM_WCSTART: ["warm", "WARM", "cold", "COLD"] +valid_vals_COMPILER: ["intel", "gnu"] +valid_vals_SUB_HOURLY_POST: [True, False] +valid_vals_DT_SUBHOURLY_POST_MNTS: [0, 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30] +valid_vals_USE_MERRA_CLIMO: [True, False] From 4f47df4f25eaf050e83815052d882f65cffe0109 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:22:06 -0400 Subject: [PATCH 158/203] Fix typo and crontab issue on wcoss dell in workflow python scripts (#750) * Fix typo and failure on wcoss * fix new line issue on wcoss dell * remove capture_output * Get USER from environment Co-authored-by: Daniel Abdi --- ush/generate_FV3LAM_wflow.py | 2 +- ush/get_crontab_contents.py | 6 +++--- ush/retrieve_data.py | 1 - ush/setup.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 16a2a63425..2391340c40 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -512,7 +512,7 @@ def generate_FV3LAM_wflow(): CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE) if MACHINE == "WCOSS_DELL_P3": - run_command(f'''printf "%s" '{CRONTAB_LINE}' >> f"/u/{USER}/cron/mycrontab"''') + run_command(f'''printf "%s\n" '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''') else: # Add a newline to the end of crontab_contents only if it is not empty. # This is needed so that when CRONTAB_LINE is printed out, it appears on diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 8316e7c7de..cbb434c69e 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -38,9 +38,9 @@ def get_crontab_contents(called_from_cron): print_input_args(locals()) #import all env vars - IMPORTS = ["MACHINE"] + IMPORTS = ["MACHINE", "USER"] import_vars(env_vars=IMPORTS) - + # # Make sure called_from_cron is set to a valid value. # @@ -79,5 +79,5 @@ def test_get_crontab_contents(self): self.assertEqual(crontab_cmd, "crontab") def setUp(self): define_macos_utilities(); - set_env_var('DEBUG','FALSE') + set_env_var('DEBUG',False) set_env_var('MACHINE', 'HERA') diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 86cb87e07b..0b65597bee 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -716,7 +716,6 @@ def parse_args(): output = subprocess.run('which hsi', check=True, shell=True, - capture_output=True, ) except subprocess.CalledProcessError: logging.error('You requested the hpss data store, but ' \ diff --git a/ush/setup.py b/ush/setup.py index 3584656ebf..cd90f70661 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -428,7 +428,7 @@ def setup(): # #----------------------------------------------------------------------- global USE_MERRA_CLIMO - if USE_MERRA_CLIMO == "FV3_GFS_v15_thompson_mynn_lam3km": + if CCPP_PHYS_SUITE == "FV3_GFS_v15_thompson_mynn_lam3km": USE_MERRA_CLIMO=True # #----------------------------------------------------------------------- From e07b8867c06a66b1b81eae911c6f2c12695737df Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Thu, 28 Apr 2022 15:46:18 -0500 Subject: [PATCH 159/203] Add new WE2E configs (#748) ## DESCRIPTION OF CHANGES: Added two new WE2E config files for the Sub-CONUS Indianapolis domain to support the upcoming SRW release. In addition, modified the external data used in the `config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh` to match more common datasets used in the WE2E testing process. ## TESTS CONDUCTED: Successfully ran the new WE2E tests (`config.SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh`, `config.SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh`) and `config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh` on NOAA Parallel Works AWS instance. ## DEPENDENCIES: None. ## DOCUMENTATION: No documentation changes are required. --- ...m_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} | 0 ...US_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 27 +++++++++++++++++++ ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 27 +++++++++++++++++++ ...g.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh | 6 ++--- 4 files changed, 57 insertions(+), 3 deletions(-) rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh => config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh} (100%) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh similarity index 100% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh new file mode 100644 index 0000000000..aa2cf6edcf --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -0,0 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the SUBCONUS_Ind_3km grid using the HRRR +# physics suite with ICs derived from HRRR and LBCs derived from the RAP. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="SUBCONUS_Ind_3km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..82c5ef43ab --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -0,0 +1,27 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the SUBCONUS_Ind_3km grid using the RRFS_v1beta +# physics suite with ICs derived from HRRR and LBCs derived from the RAP. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="SUBCONUS_Ind_3km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="RAP" +USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) + +DATE_FIRST_CYCL="20200801" +DATE_LAST_CYCL="20200801" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh index 2ade79ff57..eca762a946 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh @@ -18,9 +18,9 @@ FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -DATE_FIRST_CYCL="20210603" -DATE_LAST_CYCL="20210603" -CYCL_HRS=( "06" ) +DATE_FIRST_CYCL="20210615" +DATE_LAST_CYCL="20210615" +CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" From 7e85d73490e75c03f2129194420db9e1b2ac93dd Mon Sep 17 00:00:00 2001 From: Yunheng Wang <47898913+ywangwof@users.noreply.github.com> Date: Fri, 29 Apr 2022 10:24:11 -0500 Subject: [PATCH 160/203] Added a fixed WoF grid and the python tool to determine the write component parameters (#733) * Added a fixed WoF grid and the python tool to determine the write component parameters * Update set_predef_grid_params.sh * Renamed file as recommended and removed unused lines * Modified comment Co-authored-by: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Co-authored-by: WYH@MBP --- ush/python_utils/fv3write_parms_lambert.py | 91 ++++++++++++++++++++++ ush/set_predef_grid_params.sh | 62 ++++++++++++++- ush/valid_param_vals.sh | 3 +- 3 files changed, 151 insertions(+), 5 deletions(-) create mode 100755 ush/python_utils/fv3write_parms_lambert.py diff --git a/ush/python_utils/fv3write_parms_lambert.py b/ush/python_utils/fv3write_parms_lambert.py new file mode 100755 index 0000000000..18b4a8f359 --- /dev/null +++ b/ush/python_utils/fv3write_parms_lambert.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# +# To use this tool, you should source the regional workflow environment +# $> source env/wflow_xxx.env +# and activate pygraf (or any one with cartopy installation) +# $> conda activate pygraf +# + +import argparse + +import cartopy.crs as ccrs + +#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +# +# Main function to return parameters for the FV3 write component. +# + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Determine FV3 write component lat1/lon1 for Lamert Conformal map projection', + epilog=''' ---- Yunheng Wang (2021-07-15). + ''') + #formatter_class=CustomFormatter) + parser.add_argument('-v','--verbose', help='Verbose output', action="store_true") + parser.add_argument('-ca','--ctrlat', help='Lambert Conformal central latitude', type=float, default=38.5 ) + parser.add_argument('-co','--ctrlon', help='Lambert Conformal central longitude', type=float, default=-97.5 ) + parser.add_argument('-s1','--stdlat1',help='Lambert Conformal standard latitude1', type=float, default=38.5 ) + parser.add_argument('-s2','--stdlat2',help='Lambert Conformal standard latitude2', type=float, default=38.5 ) + parser.add_argument('-nx', help='number of grid in X direction', type=int, default=301 ) + parser.add_argument('-ny' ,help='number of grid in Y direction', type=int, default=301 ) + parser.add_argument('-dx' ,help='grid resolution in X direction (meter)',type=float, default=3000.0) + parser.add_argument('-dy' ,help='grid resolution in Y direction (meter)',type=float, default=3000.0) + + args = parser.parse_args() + + if args.verbose: + print("Write component Lambert Conformal Parameters:") + print(f" cen_lat = {args.ctrlat}, cen_lon = {args.ctrlon}, stdlat1 = {args.stdlat1}, stdlat2 = {args.stdlat2}") + print(f" nx = {args.nx}, ny = {args.ny}, dx = {args.dx}, dy = {args.dy}") + + #----------------------------------------------------------------------- + # + # Lambert grid + # + #----------------------------------------------------------------------- + + nx1 = args.nx + ny1 = args.ny + dx1 = args.dx + dy1 = args.dy + + ctrlat = args.ctrlat + ctrlon = args.ctrlon + + xctr = (nx1-1)/2*dx1 + yctr = (ny1-1)/2*dy1 + + carr= ccrs.PlateCarree() + + proj1=ccrs.LambertConformal(central_longitude=ctrlon, central_latitude=ctrlat, + false_easting=xctr, false_northing= yctr, secant_latitudes=None, + standard_parallels=(args.stdlat1, args.stdlat2), globe=None) + + lonlat1 = carr.transform_point(0.0,0.0,proj1) + + if args.verbose: + print() + print(f' lat1 = {lonlat1[1]}, lon1 = {lonlat1[0]}') + print('\n') + + #----------------------------------------------------------------------- + # + # Output write component parameters + # + #----------------------------------------------------------------------- + + print() + print("output_grid: 'lambert_conformal'") + print(f"cen_lat: {args.ctrlat}") + print(f"cen_lon: {args.ctrlon}") + print(f"stdlat1: {args.stdlat1}") + print(f"stdlat2: {args.stdlat2}") + print(f"nx: {args.nx}") + print(f"ny: {args.ny}") + print(f"dx: {args.dx}") + print(f"dy: {args.dy}") + print(f"lat1: {lonlat1[1]}") + print(f"lon1: {lonlat1[0]}") + print() + + # End of program diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index d14e9381f7..1a5e398f23 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -251,7 +251,7 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NY="232" ESGgrid_PAZI="0.0" - + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-45}" @@ -389,7 +389,7 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NY="600" ESGgrid_PAZI="0.0" - + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="${DT_ATMOS:-40}" @@ -486,7 +486,7 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NY="240" ESGgrid_PAZI="0.0" - + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" @@ -606,7 +606,7 @@ case ${PREDEF_GRID_NAME} in ESGgrid_NY="1020" ESGgrid_PAZI="0.0" - + ESGgrid_WIDE_HALO_WIDTH="6" # DT_ATMOS="${DT_ATMOS:-50}" @@ -635,6 +635,60 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # +# The WoFS domain with ~3km cells. +# +# Note: +# The WoFS domain will generate a 301 x 301 output grid (WRITE COMPONENT) and +# will eventually be movable (ESGgrid_LON_CTR/ESGgrid_LAT_CTR). A python script +# python_utils/fv3write_parms_lambert will be useful to determine +# WRTCMP_lon_lwr_left and WRTCMP_lat_lwr_left locations (only for Lambert map +# projection currently) of the quilting output when the domain location is +# moved. Later, it should be integrated into the workflow. +# +#----------------------------------------------------------------------- +# +"WoFS_3km") + + GRID_GEN_METHOD="ESGgrid" + + ESGgrid_LON_CTR="-97.5" + ESGgrid_LAT_CTR="38.5" + + ESGgrid_DELX="3000.0" + ESGgrid_DELY="3000.0" + + ESGgrid_NX="361" + ESGgrid_NY="361" + + ESGgrid_PAZI="0.0" + + ESGgrid_WIDE_HALO_WIDTH="6" + + DT_ATMOS="${DT_ATMOS:-20}" + + LAYOUT_X="${LAYOUT_X:-18}" + LAYOUT_Y="${LAYOUT_Y:-12}" + BLOCKSIZE="${BLOCKSIZE:-30}" + + if [ "$QUILTING" = "TRUE" ]; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_output_grid="lambert_conformal" + WRTCMP_cen_lon="${ESGgrid_LON_CTR}" + WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" + WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + WRTCMP_nx="301" + WRTCMP_ny="301" + WRTCMP_lon_lwr_left="-102.3802487" + WRTCMP_lat_lwr_left="34.3407918" + WRTCMP_dx="${ESGgrid_DELX}" + WRTCMP_dy="${ESGgrid_DELY}" + fi + ;; +# +#----------------------------------------------------------------------- +# # A CONUS domain of GFDLgrid type with ~25km cells. # # Note: diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a22e51d540..69c08984e1 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -31,6 +31,7 @@ valid_vals_PREDEF_GRID_NAME=( \ "RRFS_NA_13km" \ "RRFS_NA_3km" \ "SUBCONUS_Ind_3km" \ +"WoFS_3km" \ ) valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GFS_2017_gfdlmp" \ @@ -41,7 +42,7 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_RRFS_v1beta" \ "FV3_RRFS_v1alpha" \ "FV3_HRRR" \ -) +) valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") From 42a256c38ea12d7b87aadf05a15502435a02835b Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 2 May 2022 11:35:03 -0400 Subject: [PATCH 161/203] Replace env with modulefiles in scripts (#752) * change env to mod * update we2e script --- tests/WE2E/run_WE2E_tests.sh | 14 +++++++------- ush/config_defaults.sh | 12 ++++++------ ush/config_defaults.yaml | 12 ++++++------ ush/launch_FV3LAM_wflow.sh | 8 ++++---- ush/load_modules_run_task.sh | 15 ++++++++------- ush/setup.py | 8 ++++---- ush/setup.sh | 6 +++--- 7 files changed, 38 insertions(+), 37 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index bcc3b0a6a6..0adf6d7377 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -94,7 +94,7 @@ Usage: [stmp=\"...\"] \\ [ptmp=\"...\"] \\ [compiler=\"...\"] \\ - [build_env_fn=\"...\"] + [build_mod_fn=\"...\"] The arguments in brackets are optional. The arguments are defined as follows: @@ -211,9 +211,9 @@ compiler: Type of compiler to use for the workflow. Options are \"intel\" and \"gnu\". Default is \"intel\", -build_env_fn: -Specify the build environment (see ufs-srweather-app/envs) to -use for the workflow. (e.g. build_cheyenne_gnu.env). If a +build_mod_fn: +Specify the build module files (see ufs-srweather-app/modulefiles) to +use for the workflow. (e.g. build_cheyenne_gnu). If a \"gnu\" compiler is specified, it must also be specified with the \"compiler\" option. " @@ -253,7 +253,7 @@ valid_args=( \ "stmp" \ "ptmp" \ "compiler" \ - "build_env_fn" \ + "build_mod_fn" \ ) process_args valid_args "$@" # @@ -685,7 +685,7 @@ Please correct and rerun." MACHINE="${machine^^}" ACCOUNT="${account}" COMPILER=${compiler:-"intel"} - BUILD_ENV_FN=${build_env_fn:-"build_${machine}_${COMPILER}.env"} + BUILD_MOD_FN=${build_mod_fn:-"build_${machine}_${COMPILER}"} EXPT_BASEDIR="${expt_basedir}" EXPT_SUBDIR="${test_name}" USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} @@ -710,7 +710,7 @@ MACHINE=\"${MACHINE}\" ACCOUNT=\"${ACCOUNT}\" COMPILER=\"${COMPILER}\" -BUILD_ENV_FN=\"${BUILD_ENV_FN}\"" +BUILD_MOD_FN=\"${BUILD_MOD_FN}\"" if [ -n "${exec_subdir}" ]; then expt_config_str=${expt_config_str}" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index ba57dca9d9..aa6cddc368 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -73,12 +73,12 @@ RUN_ENVIR="nco" # Path to the LMOD sh file on your Linux system. Is set automatically # for supported machines. # -# BUILD_ENV_FN: -# Name of alternative build environment file to use if using an +# BUILD_MOD_FN: +# Name of alternative build module file to use if using an # unsupported platform. Is set automatically for supported machines. # -# WFLOW_ENV_FN: -# Name of alternative workflow environment file to use if using an +# WFLOW_MOD_FN: +# Name of alternative workflow module file to use if using an # unsupported platform. Is set automatically for supported machines. # # SCHED: @@ -142,8 +142,8 @@ ACCOUNT="project_name" WORKFLOW_MANAGER="none" NCORES_PER_NODE="" LMOD_PATH="" -BUILD_ENV_FN="" -WFLOW_ENV_FN="" +BUILD_MOD_FN="" +WFLOW_MOD_FN="" SCHED="" PARTITION_DEFAULT="" QUEUE_DEFAULT="" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index ac35529530..a2a218a7f8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -73,12 +73,12 @@ RUN_ENVIR: "nco" # Path to the LMOD sh file on your Linux system. Is set automatically # for supported machines. # -# BUILD_ENV_FN: -# Name of alternative build environment file to use if using an +# BUILD_MOD_FN: +# Name of alternative build module file to use if using an # unsupported platform. Is set automatically for supported machines. # -# WFLOW_ENV_FN: -# Name of alternative workflow environment file to use if using an +# WFLOW_MOD_FN: +# Name of alternative workflow module file to use if using an # unsupported platform. Is set automatically for supported machines. # # SCHED: @@ -142,8 +142,8 @@ ACCOUNT: "project_name" WORKFLOW_MANAGER: "none" NCORES_PER_NODE: "" LMOD_PATH: "" -BUILD_ENV_FN: "" -WFLOW_ENV_FN: "" +BUILD_MOD_FN: "" +WFLOW_MOD_FN: "" SCHED: "" PARTITION_DEFAULT: "" QUEUE_DEFAULT: "" diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index ddee812fa7..0971d5a898 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -149,11 +149,11 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -env_fp="${SR_WX_APP_TOP_DIR}/env/${WFLOW_ENV_FN}" -source "${env_fp}" || print_err_msg_exit "\ -Sourcing platform-specific environment file (env_fp) for the workflow +module use "${SR_WX_APP_TOP_DIR}/modulefiles" +module load "${WFLOW_MOD_FN}" || print_err_msg_exit "\ +Loading platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: - env_fp = \"${env_fp}\"" + WFLOW_MOD_FN = \"${WFLOW_MOD_FN}\"" # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 56c005faa3..7a9547fe03 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -86,18 +86,19 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # -# Sourcing ufs-srweather-app build env file +# Loading ufs-srweather-app build module files # #----------------------------------------------------------------------- # machine=$(echo_lowercase $MACHINE) -env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" -module use "${SR_WX_APP_TOP_DIR}/env" -source "${env_fp}" || print_err_msg_exit "\ -Sourcing platform- and compiler-specific environment file (env_fp) for the + +source "${SR_WX_APP_TOP_DIR}/etc/lmod-setup.sh" +module use "${SR_WX_APP_TOP_DIR}/modulefiles" +module load "${BUILD_MOD_FN}" || print_err_msg_exit "\ +Sourcing platform- and compiler-specific module file (BUILD_MOD_FN) for the workflow task specified by task_name failed: task_name = \"${task_name}\" - env_fp = \"${env_fp}\"" + BUILD_MOD_FN = \"${BUILD_MOD_FN}\"" # #----------------------------------------------------------------------- # @@ -112,7 +113,7 @@ workflow task specified by task_name failed: # # The regional_workflow repository contains module files for the # workflow tasks in the template rocoto XML file for the FV3-LAM work- -# flow that need modules not loaded in the env_fn above. +# flow that need modules not loaded in the BUILD_MOD_FN above. # # The full path to a module file for a given task is # diff --git a/ush/setup.py b/ush/setup.py index cd90f70661..261f0e26a8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -368,17 +368,17 @@ def setup(): # #----------------------------------------------------------------------- # - # Set the names of the build and workflow environment files (if not + # Set the names of the build and workflow module files (if not # already specified by the user). These are the files that need to be # sourced before building the component SRW App codes and running various # workflow scripts, respectively. # #----------------------------------------------------------------------- # - global WFLOW_ENV_FN, BUILD_ENV_FN + global WFLOW_MOD_FN, BUILD_MOD_FN machine=lowercase(MACHINE) - WFLOW_ENV_FN=WFLOW_ENV_FN or f"wflow_{machine}.env" - BUILD_ENV_FN=BUILD_ENV_FN or f"build_{machine}_{COMPILER}.env" + WFLOW_MOD_FN=WFLOW_MOD_FN or f"wflow_{machine}" + BUILD_MOD_FN=BUILD_MOD_FN or f"build_{machine}_{COMPILER}" # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index 835ccfea48..d9bdffdf11 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -523,7 +523,7 @@ fi # #----------------------------------------------------------------------- # -# Set the names of the build and workflow environment files (if not +# Set the names of the build and workflow module files (if not # already specified by the user). These are the files that need to be # sourced before building the component SRW App codes and running various # workflow scripts, respectively. @@ -531,8 +531,8 @@ fi #----------------------------------------------------------------------- # machine=$(echo_lowercase ${MACHINE}) -WFLOW_ENV_FN=${WFLOW_ENV_FN:-"wflow_${machine}.env"} -BUILD_ENV_FN=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} +WFLOW_MOD_FN=${WFLOW_MOD_FN:-"wflow_${machine}"} +BUILD_MOD_FN=${BUILD_MOD_FN:-"build_${machine}_${COMPILER}"} # #----------------------------------------------------------------------- # From fc435cb57869b644d9568b8ddd02040a4e1e1880 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 2 May 2022 11:37:49 -0600 Subject: [PATCH 162/203] WE2E script improvements for usability (#745) ## DESCRIPTION OF CHANGES: * Modifications to `run_WE2E_tests.sh`: * Add examples to help/usage statement * Modifications to `check_expts_status.sh`: * Add arguments list that can be processed by `process_args` * Add new optional arguments: `num_log_lines`, `verbose` * Include a help/usage message ## TESTS CONDUCTED: * Ran `run_WE2E_tests.sh --help` from the command line and got the expected help message. * Ran `check_expts_status.sh --help` from the command line and got the expected help message. * Used `run_WE2E_tests.sh` to run a set of 2 WE2E tests -- works as expected. * Used `check_expts_status` to check on the status of the 2 tests run above and got the expected status message. ## DEPENDENCIES: PR #[241](https://github.com/ufs-community/ufs-srweather-app/pull/241) ## DOCUMENTATION: A lot of this PR is documentation in the scripts. There is an accompanying documentation PR #[241](https://github.com/ufs-community/ufs-srweather-app/pull/241) into ufs-srweather-app. --- tests/WE2E/get_expts_status.sh | 157 +++++++++++++++++++++++++-------- tests/WE2E/run_WE2E_tests.sh | 81 +++++++++++++++-- 2 files changed, 195 insertions(+), 43 deletions(-) diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 01b127d9fe..91de215d25 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -12,8 +12,8 @@ # directory represent active experiments (see below for how this is done). # For all such experiments, it calls the workflow (re)launch script to # update the status of the workflow and prints the status out to screen. -# It also generates a summary status file in the base directory that -# contains the last num_tail_lines lines (defined below) of each experiment's +# It also generates a status report file in the base directory that +# contains the last num_log_lines lines (defined below) of each experiment's # workflow log file [which is generated by the (re)launch script] and thus # has information on which tasks may have succeeded/failed]. # @@ -70,42 +70,112 @@ ushdir="$homerrfs/ush" # #----------------------------------------------------------------------- # -# Exactly one argument must be specified that consists of the full path -# to the experiments base directory (i.e. the directory containing the -# experiment subdirectories). Ensure that the number of arguments is -# one. +# Set the usage message. # #----------------------------------------------------------------------- # -num_args="$#" -if [ "${num_args}" -eq 1 ]; then - expts_basedir="$1" -else - print_err_msg_exit " -The number of arguments to this script must be exacty one, and that -argument must specify the experiments base directory, i.e. the directory -containing the experiment subdirectories. The acutal number of arguments -is: - num_args = ${num_args}" +usage_str="\ +Usage: + + ${scrfunc_fn} \\ + expts_basedir=\"...\" \\ + [verbose=\"...\"] + +The arguments in brackets are optional. The arguments are defined as +follows: + +expts_basedir: +Full path to the experiments base directory, i.e. the directory containing +the experiment subdirectories. + +num_log_lines: +Optional integer specifying the number of lines from the end of the +workflow launch log file (log.launch_FV3LAM_wflow) of each test to +include in the status report file that this script generates. + +verbose: +Optional verbosity flag. Should be set to \"TRUE\" or \"FALSE\". Default +is \"FALSE\". +" +# +#----------------------------------------------------------------------- +# +# Check to see if usage help for this script is being requested. If so, +# print it out and exit with a 0 exit code (success). +# +#----------------------------------------------------------------------- +# +help_flag="--help" +if [ "$#" -eq 1 ] && [ "$1" = "${help_flag}" ]; then + print_info_msg "${usage_str}" + exit 0 +fi +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script or function. +# Then process the arguments provided to it on the command line (which +# should consist of a set of name-value pairs of the form arg1="value1", +# arg2="value2", etc). +# +#----------------------------------------------------------------------- +# +valid_args=( \ + "expts_basedir" \ + "num_log_lines" \ + "verbose" \ + ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# Set the default value of "num_log_lines". +# +#----------------------------------------------------------------------- +# +num_log_lines=${num_log_lines:-"40"} +# +#----------------------------------------------------------------------- +# +# Make the default value of "verbose" "FALSE". Then make sure "verbose" +# is set to a valid value. +# +#----------------------------------------------------------------------- +# +verbose=${verbose:-"FALSE"} +check_var_valid_value "verbose" "valid_vals_BOOLEAN" +verbose=$(boolify $verbose) +# +#----------------------------------------------------------------------- +# +# Verify that the required arguments to this script have been specified. +# If not, print out an error message and exit. +# +#----------------------------------------------------------------------- +# +help_msg="\ +Use + ${scrfunc_fn} ${help_flag} +to get help on how to use this script." + +if [ -z "${expts_basedir}" ]; then + print_err_msg_exit "\ +The argument \"expts_basedir\" specifying the base directory containing +the experiment directories was not specified in the call to this script. \ +${help_msg}" fi # #----------------------------------------------------------------------- # # Check that the specified experiments base directory exists and is # actually a directory. If not, print out an error message and exit. -# If so, print out an informational message. # #----------------------------------------------------------------------- # if [ ! -d "${expts_basedir}" ]; then print_err_msg_exit " -The experiments base directory (expts_basedir) does not exit or is not -actually a directory: - expts_basedir = \"${expts_basedir}\"" -else - print_info_msg " -Checking the workflow status of all forecast experiments in the following -specified experiments base directory: +The specified experiments base directory (expts_basedir) does not exit +or is not actually a directory: expts_basedir = \"${expts_basedir}\"" fi # @@ -116,7 +186,7 @@ fi # #----------------------------------------------------------------------- # -cd "${expts_basedir}" +cd_vrfy "${expts_basedir}" # # Get a list of all subdirectories (but not files) in the experiment base # directory. Note that the ls command below will return a string containing @@ -175,6 +245,12 @@ var_defns_fn="var_defns.sh" j="0" expt_subdirs=() +print_info_msg "\ +Checking for active experiment directories in the specified experiments +base directory (expts_basedir): + expts_basedir = \"${expts_basedir}\" +..." + num_subdirs="${#subdirs_list[@]}" for (( i=0; i<=$((num_subdirs-1)); i++ )); do @@ -184,7 +260,7 @@ $separator Checking whether the subdirectory \"${subdir}\" contains an active experiment..." - print_info_msg "$msg" + print_info_msg "$verbose" "$msg" cd_vrfy "${subdir}" # @@ -193,7 +269,7 @@ contains an active experiment..." # if [ ! -f "${var_defns_fn}" ]; then - print_info_msg " + print_info_msg "$verbose" " The current subdirectory (subdir) under the experiments base directory (expts_basedir) does not contain an experiment variable defintions file (var_defns_fn): @@ -219,7 +295,7 @@ must be checked." # if [ "${EXPT_SUBDIR}" = "$subdir" ]; then - print_info_msg " + print_info_msg "$verbose" " The current subdirectory (subdir) under the experiments base directory (expts_basedir) contains an active experiment: expts_basedir = \"${expts_basedir}\" @@ -238,7 +314,7 @@ subdirectories whose workflow status must be checked." # else - print_info_msg " + print_info_msg "$verbose" " The current subdirectory (subdir) under the experiments base directory (expts_basedir) contains an experiment whose original name (EXPT_SUBDIR) does not match the name of the current subdirectory: @@ -254,7 +330,7 @@ status must be checked." fi - print_info_msg "\ + print_info_msg "$verbose" "\ $separator " # @@ -302,7 +378,7 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename" # Loop through the elements of the array expt_subdirs. For each element # (i.e. for each active experiment), change location to the experiment # directory and call the script launch_FV3LAM_wflow.sh to update the log -# file log.launch_FV3LAM_wflow. Then take the last num_tail_lines of +# file log.launch_FV3LAM_wflow. Then take the last num_log_lines of # this log file (along with an appropriate message) and add it to the # status report file. # @@ -310,7 +386,6 @@ check_for_preexist_dir_file "${expts_status_fp}" "rename" # launch_wflow_fn="launch_FV3LAM_wflow.sh" launch_wflow_log_fn="log.launch_FV3LAM_wflow" -num_tail_lines="40" for (( i=0; i<=$((num_expts-1)); i++ )); do @@ -326,25 +401,28 @@ Checking workflow status of experiment \"${expt_subdir}\" ..." # cd_vrfy "${expt_subdir}" launch_msg=$( "${launch_wflow_fn}" 2>&1 ) - log_tail=$( tail -n ${num_tail_lines} "${launch_wflow_log_fn}" ) + log_tail=$( tail -n ${num_log_lines} "${launch_wflow_log_fn}" ) # # Print the workflow status to the screen. # - wflow_status=$( printf "${log_tail}" | grep "Workflow status:" ) -# wflow_status="${wflow_status## }" # Not sure why this doesn't work to strip leading spaces. - wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) # Remove leading spaces. + # The "tail -1" is to get only the last occurrence of "Workflow status" + wflow_status=$( printf "${log_tail}" | grep "Workflow status:" | tail -1 ) + # Not sure why this doesn't work to strip leading spaces. +# wflow_status="${wflow_status## }" + # Remove leading spaces. + wflow_status=$( printf "${wflow_status}" "%s" | sed -r 's|^[ ]*||g' ) print_info_msg "${wflow_status}" print_info_msg "\ $separator " # -# Combine message above with the last num_tail_lines lines from the workflow +# Combine message above with the last num_log_lines lines from the workflow # launch log file and place the result in the status report file. # msg=$msg" ${wflow_status} -The last ${num_tail_lines} lines of this experiment's workflow launch log file +The last ${num_log_lines} lines of this experiment's workflow launch log file (\"${launch_wflow_log_fn}\") are: ${log_tail} @@ -360,4 +438,7 @@ ${log_tail} done print_info_msg "\ +A status report has been created in: + expts_status_fp = \"${expts_status_fp}\" + DONE." diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 0adf6d7377..330669c5ee 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -133,9 +133,9 @@ tests under subdirectory testset1, another set of tests under testset2, etc. exec_subdir: -Optional. Argument is used to set the EXEC_SUBDIR configuration -variable. Please see the ush/default_configs.sh file for a full -description. +Optional. Argument used to set the EXEC_SUBDIR experiment variable. +Please see the default experiment configuration file \"config_defaults.sh\" +for a full description of EXEC_SUBDIR. use_cron_to_relaunch: Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH @@ -208,14 +208,85 @@ Same as the argument \"stmp\" described above but for setting the experiment variable PTMP for all tests that will run in NCO mode. compiler: -Type of compiler to use for the workflow. Options are \"intel\" -and \"gnu\". Default is \"intel\", +Type of compiler to use for the workflow. Options are \"intel\" and \"gnu\". +Default is \"intel\". build_mod_fn: Specify the build module files (see ufs-srweather-app/modulefiles) to use for the workflow. (e.g. build_cheyenne_gnu). If a \"gnu\" compiler is specified, it must also be specified with the \"compiler\" option. + + +Usage Examples: +-------------- +Here, we give several common usage examples. In the following, assume +my_tests.txt is a text file in the same directory as this script containing +a list of test names that we want to run, e.g. + +> more my_tests.txt +new_ESGgrid +specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE + +Then: + +1) To run the tests listed in my_tests.txt on Hera and charge the core- + hours used to the \"rtrr\" account, use: + + > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" + + This will create the experiment subdirectories for the two tests in + the directory + + \${SR_WX_APP_TOP_DIR}/../expt_dirs + + where SR_WX_APP_TOP_DIR is the directory in which the ufs-srweather-app + repository is cloned. Thus, the following two experiment directories + will be created: + + \${SR_WX_APP_TOP_DIR}/../expt_dirs/new_ESGgrid + \${SR_WX_APP_TOP_DIR}/../expt_dirs/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE + + In addition, by default, cron jobs will be created in the user's cron + table to relaunch the workflows of these experiments every 2 minutes. + +2) To change the frequency with which the cron relaunch jobs are submitted + from the default of 2 minutes to 1 minute, use: + + > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" cron_relaunch_intvl_mnts=\"01\" + +3) To disable use of cron (which means the worfkow for each test will + have to be relaunched manually from within each experiment directory), + use: + + > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" use_cron_to_relaunch=\"FALSE\" + +4) To place the experiment subdirectories in a subdirectory named \"test_set_01\" + under + + \${SR_WX_APP_TOP_DIR}/../expt_dirs + + (instead of immediately under the latter), use: + + > run_WE2E_tests.sh tests_file=\"my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"test_set_01\" + + In this case, the full paths to the experiment directories will be: + + \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/new_ESGgrid + \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE + +5) To use a list of tests that is located in + + /path/to/custom/my_tests.txt + + instead of in the same directory as this script, and to have the + experiment directories be placed in an arbitrary location, say + + /path/to/custom/expt_dirs + + use: + + > run_WE2E_tests.sh tests_file=\"/path/to/custom/my_tests.txt\" machine=\"hera\" account=\"rtrr\" expt_basedir=\"/path/to/custom/expt_dirs\" " # #----------------------------------------------------------------------- From 543a94e1a429911c29c4dd704ef51c21b552bea3 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Tue, 3 May 2022 13:53:10 -0600 Subject: [PATCH 163/203] Standardize static data across Tier-1 platforms; fix and improve IC and LBC data retrieval (#744) --- scripts/exregional_get_extrn_mdl_files.sh | 4 +- tests/WE2E/run_WE2E_tests.sh | 22 +++++----- ...NUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh | 1 + ...mpact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...pact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh | 3 ++ ...km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh | 2 + ...mpact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...5km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...t_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 2 + ...ompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 3 ++ ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 2 + ...3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh | 2 + ...S_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh | 2 + ...mpact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh | 2 + .../config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh | 2 + ...ig.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh | 2 + .../wflow_features/config.subhourly_post.sh | 2 + .../config.subhourly_post_ensemble_2mems.sh | 2 + ush/NOMADS_get_extrn_mdl_files.sh | 12 +++--- ush/config_defaults.sh | 41 ++++++++++++------- ush/machine/cheyenne.sh | 25 +++++------ ush/machine/gaea.sh | 14 +++---- ush/machine/hera.sh | 27 ++++++------ ush/machine/jet.sh | 21 ++++++---- ush/machine/noaacloud.sh | 33 ++++++++------- ush/machine/odin.sh | 32 +++++++++++---- ush/machine/orion.sh | 27 ++++++------ ush/machine/singularity.sh | 4 +- ush/machine/stampede.sh | 34 +++++++++++---- ush/machine/wcoss_dell_p3.sh | 27 ++++++------ ush/retrieve_data.py | 2 +- ush/set_extrn_mdl_params.sh | 17 -------- ush/setup.sh | 23 ++++++----- ush/templates/data_locations.yml | 2 - 37 files changed, 242 insertions(+), 164 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index fc02e3b996..5a3fe04cf3 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -108,7 +108,7 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi -data_stores="hpss aws" +data_stores="${EXTRN_MDL_DATA_STORES}" yyyymmddhh=${extrn_mdl_cdate:0:10} yyyy=${yyyymmddhh:0:4} @@ -141,7 +141,7 @@ if [ -n "${file_names:-}" ] ; then fi if [ -n "${input_file_path:-}" ] ; then - data_stores="disk hpss aws" + data_stores="disk $data_stores" additional_flags="$additional_flags \ --input_file_path ${input_file_path}" fi diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 330669c5ee..a11dfaba3a 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -995,6 +995,9 @@ PTMP=\"${PTMP}\"" # if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then + # Ensure we only check on disk for these files + data_stores="disk" + extrn_mdl_source_basedir=${TEST_EXTRN_MDL_SOURCE_BASEDIR:-} if [ ! -d "${extrn_mdl_source_basedir:-}" ] ; then print_err_msg_exit "\ @@ -1005,20 +1008,16 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_ICS" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${yyyymmddhh}" else - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${yyyymmddhh}" fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" - elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${yyyymmddhh}" else - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${yyyymmddhh}" fi # # Make sure that the forecast length is evenly divisible by the interval @@ -1039,10 +1038,11 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS} +EXTRN_MDL_SOURCE_BASEDIR_ICS='${EXTRN_MDL_SOURCE_BASEDIR_ICS}' EXTRN_MDL_FILES_ICS=( ${EXTRN_MDL_FILES_ICS[@]} ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS} -EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} )" +EXTRN_MDL_SOURCE_BASEDIR_LBCS='${EXTRN_MDL_SOURCE_BASEDIR_LBCS}' +EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} ) +EXTRN_MDL_DATA_STORES=\"$data_stores\"" fi # diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh index 492a163360..1465641a59 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh @@ -22,6 +22,7 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 4239752de4..40f5e4997e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 8aaeddf43c..f44afffcda 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 9e86198e6f..afcfa32e16 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh index 67b381044c..f1302d1634 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" @@ -23,3 +25,4 @@ CYCL_HRS=( "00" ) FCST_LEN_HRS="24" LBC_SPEC_INTVL_HRS="3" + diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh index 09b9548d11..0060e44663 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index f4b781e070..3dfedb5686 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index b0b8a2e42d..bf2e2f15e5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 2a98569a39..8fc60571db 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 2407157518..11227ea005 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index eda52d1e7e..396ce3e150 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) + DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 1b3131663d..cc92aecaaa 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index cf6965435f..a75f8d79e6 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 3534d5df9e..c5512d0ead 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index c2a054b309..e5c4e7732d 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh index 5d6ed126ec..61d4e5c9eb 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh @@ -17,6 +17,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index d985559f2e..7a10a0ec34 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh index 09df72e29a..a1676e4d58 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh @@ -15,6 +15,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh index 4c20807119..adcc37a332 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh @@ -21,6 +21,8 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/ush/NOMADS_get_extrn_mdl_files.sh b/ush/NOMADS_get_extrn_mdl_files.sh index 308aa80827..789eeff3dc 100755 --- a/ush/NOMADS_get_extrn_mdl_files.sh +++ b/ush/NOMADS_get_extrn_mdl_files.sh @@ -37,10 +37,10 @@ cd gfs.$yyyymmdd/$hh #getting online analysis data if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f000 + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f000 else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmanl.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcanl.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmanl.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcanl.nemsio fi #getting online forecast data @@ -60,10 +60,10 @@ echo $ifcst echo $ifcst_str # if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f${ifcst_str} + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f${ifcst_str} else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmf${ifcst_str}.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcf${ifcst_str}.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmf${ifcst_str}.nemsio + wget --tries=2 -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcf${ifcst_str}.nemsio fi # ifcst=$[$ifcst+$nfcst_int] diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index aa6cddc368..8af4590a6e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -238,19 +238,6 @@ EXEC_SUBDIR="bin" # need in order to create initial and boundary condition files for a given # cycle on the native FV3-LAM grid. # -# FIXLAM_NCO_BASEDIR: -# The base directory containing pregenerated grid, orography, and surface -# climatology files. For the pregenerated grid specified by PREDEF_GRID_NAME, -# these "fixed" files are located in: -# -# ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME} -# -# The workflow scripts will create a symlink in the experiment directory -# that will point to a subdirectory (having the name of the grid being -# used) under this directory. This variable should be set to a null -# string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN). -# # envir, NET, model_ver, RUN: # Standard environment variables defined in the NCEP Central Operations WCOSS # Implementation Standards document as follows: @@ -292,7 +279,7 @@ EXEC_SUBDIR="bin" #----------------------------------------------------------------------- # COMIN="/path/of/directory/containing/data/files/for/IC/LBCS" -FIXLAM_NCO_BASEDIR="" +STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" envir="para" NET="rrfs" model_ver="v1.0.0" @@ -711,6 +698,13 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # EXTRN_MDL_FILES_LBCS: # Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. # +# EXTRN_MDL_DATA_STORES: +# A list of data stores where the scripts should look for external model +# data. The list is in priority order. If disk information is provided +# via USE_USER_STAGED_EXTRN_FILES or a known location on the platform, +# the disk location will be highest priority. Options are disk, hpss, +# aws, and nomads. +# #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES="FALSE" @@ -718,6 +712,7 @@ EXTRN_MDL_SOURCE_BASEDIR_ICS="" EXTRN_MDL_FILES_ICS="" EXTRN_MDL_SOURCE_BASEDIR_LBCS="" EXTRN_MDL_FILES_LBCS="" +EXTRN_MDL_DATA_STORES="" # #----------------------------------------------------------------------- # @@ -1305,6 +1300,22 @@ VX_ENSPOINT_PROB_TN="run_enspointvx_prob" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # +# DOMAIN_PREGEN_BASEDIR: +# The base directory containing pregenerated grid, orography, and surface +# climatology files. This is an alternative for setting GRID_DIR, +# OROG_DIR, and SFC_CLIMO_DIR individually +# +# For the pregenerated grid specified by PREDEF_GRID_NAME, +# these "fixed" files are located in: +# +# ${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME} +# +# The workflow scripts will create a symlink in the experiment directory +# that will point to a subdirectory (having the name of the grid being +# used) under this directory. This variable should be set to a null +# string in this file, but it can be specified in the user-specified +# workflow configuration file (EXPT_CONFIG_FN). +# # RUN_TASK_GET_EXTRN_ICS: # Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. # @@ -1351,6 +1362,8 @@ OROG_DIR="/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO="TRUE" SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files" +DOMAIN_PREGEN_BASEDIR="" + RUN_TASK_GET_EXTRN_ICS="TRUE" RUN_TASK_GET_EXTRN_LBCS="TRUE" RUN_TASK_MAKE_ICS="TRUE" diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index f92c6fa844..cf80db74f4 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -46,12 +46,13 @@ QUEUE_HPSS=${QUEUE_HPSS:-"regular"} QUEUE_FCST=${QUEUE_FCST:-"regular"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} -FIXaer=${FIXaer:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_aer"} -FIXlut=${FIXlut:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +staged_data_dir="/glade/p/ral/jntp/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -62,12 +63,12 @@ RUN_CMD_POST='mpirun -np $nprocs' # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/glade/p/ral/jntp/UFS_SRW_app/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/glade/p/ral/jntp/UFS_SRW_app/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 9bad0af743..37f0838e6f 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -48,11 +48,13 @@ QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} WTIME_MAKE_LBCS="00:60:00" # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"//lustre/f2/dev/Mark.Potts/EPIC/fix/fix_am"} -FIXaer=${FIXaer:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_aer"} -FIXlut=${FIXlut:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lustre/f2/dev/Mark.Potts/EPIC/fix/fix_sfc_climo"} +staged_data_dir="/lustre/f2/dev/Mark.Potts/EPIC/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" RUN_CMD_SERIAL="time" #Run Commands currently differ for GNU/openmpi @@ -66,5 +68,3 @@ RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations # MET Plus is not yet supported on gaea # Test Data Locations -TEST_EXTRN_MDL_SOURCE_BASEDIR="/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data" - diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index 39c28f8ee4..454a1e9383 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -28,6 +28,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -49,12 +51,13 @@ PARTITION_FCST=${PARTITION_FCST:-"hera"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} -FIXaer=${FIXaer:-"/scratch1/NCEPDEV/global/glopara/fix/fix_aer"} -FIXlut=${FIXlut:-"/scratch1/NCEPDEV/global/glopara/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} +staged_data_dir="/scratch2/BMC/det/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -65,14 +68,14 @@ RUN_CMD_POST="srun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/scratch2/BMC/det/UFS_SRW_app/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/scratch2/NCEPDEV/fv3-cam/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/scratch2/BMC/det/UFS_SRW_app/develop/model_data" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index 9a12c0c142..f383090ec5 100644 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -44,6 +44,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -65,12 +67,13 @@ PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} -FIXaer=${FIXaer:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_aer"} -FIXlut=${FIXlut:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen"} +staged_data_dir="/mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -79,6 +82,6 @@ RUN_CMD_FCST="srun" RUN_CMD_POST="srun" # Test Data Locations -TEST_PREGEN_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/FV3LAM_pregen" -TEST_COMIN="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/mnt/lfs4/BMC/wrfruc/UFS_SRW_app/staged_extrn_mdl_files" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/noaacloud.sh b/ush/machine/noaacloud.sh index fe42766612..49299c325f 100755 --- a/ush/machine/noaacloud.sh +++ b/ush/machine/noaacloud.sh @@ -1,6 +1,5 @@ -#!/bin/bash +#!/bin/bash -set -x function file_location() { @@ -16,16 +15,13 @@ function file_location() { "FV3GFS") location='/contrib/GST/model_data/FV3GFS/${yyyymmdd}${hh}' ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Hera. Will try to pull from HPSS" - ;; esac echo ${location:-} } +export OPT=/contrib/EPIC/hpc-modules +export PATH=${PATH}:/contrib/GST/miniconda/envs/regional_workflow/bin EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ ${EXTRN_MDL_NAME_ICS} \ @@ -34,6 +30,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -50,22 +48,23 @@ NCORES_PER_NODE=${NCORES_PER_NODE:-36} SCHED=${SCHED:-"slurm"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/contrib/EPIC/fix/fix_am"} -FIXaer=${FIXaer:-"/contrib/EPIC/fix/fix_aer"} -FIXlut=${FIXlut:-"/contrib/EPIC/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/contrib/EPIC/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/EPIC/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch2/BMC/det/FV3LAM_pregen"} +staged_data_dir="/contrib/EPIC/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" RUN_CMD_SERIAL="time" #Run Commands currently differ for GNU/openmpi #RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' #RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' #RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' -RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' -RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' -RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' +RUN_CMD_UTILS='mpiexec -np $nprocs' +RUN_CMD_FCST='mpiexec -np ${PE_MEMBER01}' +RUN_CMD_POST='mpiexec -np $nprocs' # MET Installation Locations # MET Plus is not yet supported on noaacloud - +. /contrib/EPIC/.bash_conda diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh index 1bceaa873e..58d360a633 100644 --- a/ush/machine/odin.sh +++ b/ush/machine/odin.sh @@ -9,16 +9,26 @@ function file_location() { external_model=${1} external_file_fmt=${2} + staged_data_dir="/scratch/ywang/UFS_SRW_App/develop" + location="" case ${external_model} in "GSMGFS") - location='/scratch/ywang/EPIC/GDAS/2019053000_mem001' + location="${staged_data_dir}/input_model_data/GFS" ;; "FV3GFS") - location='/scratch/ywang/test_runs/FV3_regional/gfs/${yyyymmdd}' + location="${staged_data_dir}/input_model_data/FV3GFS" + ;; + "HRRR") + location="${staged_data_dir}/input_model_data/HRRR" + ;; + "RAP") + location="${staged_data_dir}/input_model_data/RAP" + ;; + "NAM") + location="${staged_data_dir}/input_model_data/NAM" ;; - esac echo ${location:-} @@ -52,15 +62,19 @@ PARTITION_FCST=${PARTITION_FCST:-"workq"} QUEUE_FCST=${QUEUE_FCST:-"workq"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} -FIXaer=${FIXaer:-"/scratch/ywang/fix/theia_fix/fix_aer"} -FIXlut=${FIXlut:-"/scratch/ywang/fix/theia_fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="srun -n 1" RUN_CMD_UTILS='srun -n $nprocs' RUN_CMD_FCST='srun -n ${PE_MEMBER01}' RUN_CMD_POST="srun -n 1" + +# Test Data Locations +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index ce39308d4e..b8032e2412 100644 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -22,6 +22,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -43,12 +45,13 @@ PARTITION_FCST=${PARTITION_FCST:-"orion"} QUEUE_FCST=${QUEUE_FCST:-"batch"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/work/noaa/global/glopara/fix/fix_am"} -FIXaer=${FIXaer:-"/work/noaa/global/glopara/fix/fix_aer"} -FIXlut=${FIXlut:-"/work/noaa/global/glopara/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/work/noaa/global/glopara/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/work/noaa/fv3-cam/UFS_SRW_App/FIXLAM_NCO_BASE"} +staged_data_dir="/work/noaa/fv3-cam/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" @@ -59,13 +62,13 @@ RUN_CMD_POST="srun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.0"} METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} # Test Data Locations -TEST_PREGEN_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/FV3LAM_pregen" -TEST_COMIN="/work/noaa/fv3-cam/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/work/noaa/fv3-cam/UFS_SRW_App/develop/model_data" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/singularity.sh b/ush/machine/singularity.sh index 528e1dbd5c..14f840800b 100644 --- a/ush/machine/singularity.sh +++ b/ush/machine/singularity.sh @@ -21,6 +21,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -47,7 +49,7 @@ FIXaer=${FIXaer:-"/contrib/global/glopara/fix/fix_aer"} FIXlut=${FIXlut:-"/contrib/global/glopara/fix/fix_lut"} TOPO_DIR=${TOPO_DIR:-"/contrib/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/global/glopara/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"/needs/to/be/specified"} # Run commands for executables RUN_CMD_SERIAL="time" diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh index 41afa5fc17..3f879ea22f 100644 --- a/ush/machine/stampede.sh +++ b/ush/machine/stampede.sh @@ -9,15 +9,26 @@ function file_location() { external_model=${1} external_file_fmt=${2} + staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" + location="" case ${external_model} in "GSMGFS") - ;& # Fall through. All files in same place + location="${staged_data_dir}/input_model_data/GFS" + ;; "FV3GFS") - location='/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001' + location="${staged_data_dir}/input_model_data/FV3GFS" + ;; + "HRRR") + location="${staged_data_dir}/input_model_data/HRRR" + ;; + "RAP") + location="${staged_data_dir}/input_model_data/RAP" + ;; + "NAM") + location="${staged_data_dir}/input_model_data/NAM" ;; - esac echo ${location:-} @@ -51,15 +62,20 @@ PARTITION_FCST=${PARTITION_FCST:-"normal"} QUEUE_FCST=${QUEUE_FCST:-"normal"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} -FIXaer=${FIXaer:-"/work/00315/tg455890/stampede2/regional_fv3/fix_aer"} -FIXlut=${FIXlut:-"/work/00315/tg455890/stampede2/regional_fv3/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/needs/to/be/specified"} +staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="time" RUN_CMD_UTILS='ibrun -np $nprocs' RUN_CMD_FCST='ibrun -np $nprocs' RUN_CMD_POST='ibrun -np $nprocs' + +# Test Data Locations +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 0c8b7cd248..9bf525a353 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -37,6 +37,8 @@ EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ ${EXTRN_MDL_NAME_LBCS} \ ${FV3GFS_FILE_FMT_LBCS})} +EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss"} + # System scripts to source to initialize various commands within workflow # scripts (e.g. "module"). if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then @@ -55,12 +57,13 @@ QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} QUEUE_FCST=${QUEUE_FCST:-"dev"} # UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} -FIXaer=${FIXaer:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_aer"} -FIXlut=${FIXlut:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} -FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FIXLAM_NCO_BASE"} +staged_data_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop" +FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} +FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} +FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} +DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} # Run commands for executables RUN_CMD_SERIAL="mpirun" @@ -71,12 +74,12 @@ RUN_CMD_POST="mpirun" # MET/METplus-Related Paths MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0"} METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} # Test Data Locations -TEST_PREGEN_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/FV3LAM_pregen" -TEST_COMIN="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop/model_data" +TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" +TEST_COMIN="${staged_data_dir}/COMGFS" +TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 0b65597bee..d68c082370 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -578,7 +578,7 @@ def main(cla): write_summary_file(cla, data_store, file_templates) break - logging.warning(f'Requested files are unavialable from {data_store}') + logging.warning(f'Requested files are unavailable from {data_store}') if unavailable: logging.error('Could not find any of the requested files.') diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 725759dc99..12b11e88da 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -6,23 +6,6 @@ #----------------------------------------------------------------------- # function set_extrn_mdl_params() { - # - #----------------------------------------------------------------------- - # - # Use known locations or COMIN as default, depending on RUN_ENVIR - # - #----------------------------------------------------------------------- - # - if [ "${RUN_ENVIR}" = "nco" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMIN}" - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMIN}" - else - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$(set_known_sys_dir \ - ${EXTRN_MDL_NAME_ICS})}" - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$(set_known_sys_dir \ - ${EXTRN_MDL_NAME_LBCS})}" - fi - # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index d9bdffdf11..67b6776926 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -507,7 +507,7 @@ if [ -z "${NCORES_PER_NODE:-}" ]; then fi if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then - print_err_msg_exit "\ + print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} @@ -515,7 +515,7 @@ One or more fix file directories have not been specified for this machine: FIXlut = \"${FIXlut:-\"\"} TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR:-\"\"} You can specify the missing location(s) in ${machine_file}" fi @@ -1352,14 +1352,15 @@ USE_USER_STAGED_EXTRN_FILES=$(boolify $USE_USER_STAGED_EXTRN_FILES) # if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_ICS}" ]; then + # Check for the base directory up to the first templated field. + if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_ICS%%\$*})" ]; then print_err_msg_exit "\ The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: EXTRN_MDL_SOURCE_BASEDIR_ICS = \"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\"" fi - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_LBCS}" ]; then + if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_LBCS%%\$*})" ]; then print_err_msg_exit "\ The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: @@ -1474,7 +1475,7 @@ LOAD_MODULES_RUN_TASK_FP="$USHDIR/load_modules_run_task.sh" # if [ "${RUN_ENVIR}" = "nco" ]; then - nco_fix_dir="${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME}" + nco_fix_dir="${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME}" if [ ! -d "${nco_fix_dir}" ]; then print_err_msg_exit "\ The directory (nco_fix_dir) that should contain the pregenerated grid, @@ -1490,11 +1491,11 @@ orography, and surface climatology files does not exist: When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated grid files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must @@ -1528,11 +1529,11 @@ Reset values are: msg=" When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated orography files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must @@ -1567,11 +1568,11 @@ Reset values are: When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated surface climatology files already exist in the directory - \${FIXLAM_NCO_BASEDIR}/\${PREDEF_GRID_NAME} + \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index d82eeec677..ef5d49c8e7 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -64,14 +64,12 @@ FV3GFS: - gfs.t{hh}z.sfcanl.nemsio fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio - - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio netcdf: anl: - gfs.t{hh}z.atmanl.nc - gfs.t{hh}z.sfcanl.nc fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nc - - gfs.t{hh}z.sfcf{fcst_hr:03d}.nc hpss: protocol: htar archive_path: From 4b59a7eb86cfe8398ae4af62937cdc27bca6ee2d Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 5 May 2022 13:32:08 -0600 Subject: [PATCH 164/203] Bug fixes (grid size + suppress screen output from module load) (#756) ## DESCRIPTION OF CHANGES: 1) Adjust y-direction size of write-component grid of `SUBCONUS_Ind_3km` predefined grid from 195 to 197 (this was just an oversight in PR #725 ). 2) Redirect output of module load in launch script (`launch_FV3LAM_wflow.sh`) to `/dev/null` to avoid unwanted screen output (which was introduced in PR #[238](https://github.com/ufs-community/ufs-srweather-app/pull/238) in ufs-srweather-app and is about how to load the `regional_workflow` environment and is not relevant in this context). ## TESTS CONDUCTED: 1) Plotted the `SUBCONUS_Ind_3km` grid to ensure it has correct size (it does). 2) Manually ran `launch_FV3LAM_wflow.sh` from the command line to verify that screen output is suppressed (it is). --- ush/launch_FV3LAM_wflow.sh | 2 +- ush/set_predef_grid_params.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 0971d5a898..9184cb0e0c 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -150,7 +150,7 @@ expt_name="${EXPT_SUBDIR}" #----------------------------------------------------------------------- # module use "${SR_WX_APP_TOP_DIR}/modulefiles" -module load "${WFLOW_MOD_FN}" || print_err_msg_exit "\ +module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || print_err_msg_exit "\ Loading platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: WFLOW_MOD_FN = \"${WFLOW_MOD_FN}\"" diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 1a5e398f23..fd4ef530bb 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -455,7 +455,7 @@ case ${PREDEF_GRID_NAME} in WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" WRTCMP_nx="197" - WRTCMP_ny="195" + WRTCMP_ny="197" WRTCMP_lon_lwr_left="-89.47120417" WRTCMP_lat_lwr_left="37.07809642" WRTCMP_dx="${ESGgrid_DELX}" From e5d159b43952821caf05211225c309dd24c9a98b Mon Sep 17 00:00:00 2001 From: JeffBeck-NOAA <55201531+JeffBeck-NOAA@users.noreply.github.com> Date: Mon, 9 May 2022 10:44:26 -0600 Subject: [PATCH 165/203] Update default SPP ISEED array in config_defaults.sh to use unique values (#759) * Modify RRFS North America 3- and 13-km domain configuration and WE2E test. * Modify default ISEED values for SPP * Fix grid in WE2E test --- ush/config_defaults.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 8af4590a6e..7cce8c1a69 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1866,7 +1866,7 @@ SPP_TSCALE=( "21600.0" "21600.0" "21600.0" "21600.0" "21600.0" ) #Variable "spp_ SPP_SIGTOP1=( "0.1" "0.1" "0.1" "0.1" "0.1") SPP_SIGTOP2=( "0.025" "0.025" "0.025" "0.025" "0.025" ) SPP_STDDEV_CUTOFF=( "1.5" "1.5" "2.5" "1.5" "1.5" ) -ISEED_SPP=( "4" "4" "4" "4" "4" ) +ISEED_SPP=( "4" "5" "6" "7" "8" ) # #----------------------------------------------------------------------- # From 617d071393270ad3363e47265a4544d20406f808 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 9 May 2022 13:25:32 -0400 Subject: [PATCH 166/203] Update workflow python scripts (#760) * update python scripts --- ush/config_defaults.yaml | 40 ++++++++++++++++++++++++++-------------- ush/setup.py | 23 ++++++++++++++--------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index a2a218a7f8..6dbd311102 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -238,19 +238,6 @@ EXEC_SUBDIR: "bin" # need in order to create initial and boundary condition files for a given # cycle on the native FV3-LAM grid. # -# FIXLAM_NCO_BASEDIR: -# The base directory containing pregenerated grid, orography, and surface -# climatology files. For the pregenerated grid specified by PREDEF_GRID_NAME, -# these "fixed" files are located in: -# -# ${FIXLAM_NCO_BASEDIR}/${PREDEF_GRID_NAME} -# -# The workflow scripts will create a symlink in the experiment directory -# that will point to a subdirectory (having the name of the grid being -# used) under this directory. This variable should be set to a null -# string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN). -# # envir, NET, model_ver, RUN: # Standard environment variables defined in the NCEP Central Operations WCOSS # Implementation Standards document as follows: @@ -292,7 +279,6 @@ EXEC_SUBDIR: "bin" #----------------------------------------------------------------------- # COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS" -FIXLAM_NCO_BASEDIR: "" envir: "para" NET: "rrfs" model_ver: "v1.0.0" @@ -711,6 +697,13 @@ EXTRN_MDL_SYSBASEDIR_LBCS: '' # EXTRN_MDL_FILES_LBCS: # Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. # +# EXTRN_MDL_DATA_STORES: +# A list of data stores where the scripts should look for external model +# data. The list is in priority order. If disk information is provided +# via USE_USER_STAGED_EXTRN_FILES or a known location on the platform, +# the disk location will be highest priority. Options are disk, hpss, +# aws, and nomads. +# #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES: "FALSE" @@ -718,6 +711,7 @@ EXTRN_MDL_SOURCE_BASEDIR_ICS: "" EXTRN_MDL_FILES_ICS: "" EXTRN_MDL_SOURCE_BASEDIR_LBCS: "" EXTRN_MDL_FILES_LBCS: "" +EXTRN_MDL_DATA_STORES: "" # #----------------------------------------------------------------------- # @@ -1305,6 +1299,22 @@ VX_ENSPOINT_PROB_TN: "run_enspointvx_prob" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # +# DOMAIN_PREGEN_BASEDIR: +# The base directory containing pregenerated grid, orography, and surface +# climatology files. This is an alternative for setting GRID_DIR, +# OROG_DIR, and SFC_CLIMO_DIR individually +# +# For the pregenerated grid specified by PREDEF_GRID_NAME, +# these "fixed" files are located in: +# +# ${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME} +# +# The workflow scripts will create a symlink in the experiment directory +# that will point to a subdirectory (having the name of the grid being +# used) under this directory. This variable should be set to a null +# string in this file, but it can be specified in the user-specified +# workflow configuration file (EXPT_CONFIG_FN). +# # RUN_TASK_GET_EXTRN_ICS: # Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. # @@ -1351,6 +1361,8 @@ OROG_DIR: "/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO: "TRUE" SFC_CLIMO_DIR: "/path/to/pregenerated/surface/climo/files" +DOMAIN_PREGEN_BASEDIR: "" + RUN_TASK_GET_EXTRN_ICS: "TRUE" RUN_TASK_GET_EXTRN_LBCS: "TRUE" RUN_TASK_MAKE_ICS: "TRUE" diff --git a/ush/setup.py b/ush/setup.py index 261f0e26a8..acf46f1cdf 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -337,7 +337,7 @@ def setup(): # global MACHINE global MACHINE_FILE - global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, FIXLAM_NCO_BASEDIR, \ + global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, DOMAIN_PREGEN_BASEDIR, \ RELATIVE_LINK_FLAG, WORKFLOW_MANAGER, NCORES_PER_NODE, SCHED, \ QUEUE_DEFAULT, QUEUE_HPSS, QUEUE_FCST, \ PARTITION_DEFAULT, PARTITION_HPSS, PARTITION_FCST @@ -362,7 +362,7 @@ def setup(): FIXlut = \"{FIXlut or ""} TOPO_DIR = \"{TOPO_DIR or ""} SFC_CLIMO_INPUT_DIR = \"{SFC_CLIMO_INPUT_DIR or ""} - FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR or ""} + DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR or ""} You can specify the missing location(s) in config.sh''') # @@ -1042,12 +1042,17 @@ def setup(): if USE_USER_STAGED_EXTRN_FILES: if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS): + # Check for the base directory up to the first templated field. + idx = EXTRN_MDL_SOURCE_BASEDIR_ICS.find("$") + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]): print_err_msg_exit(f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''') if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS): + idx = EXTRN_MDL_SOURCE_BASEDIR_LBCS.find("$") + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): print_err_msg_exit(f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: @@ -1153,7 +1158,7 @@ def setup(): if RUN_ENVIR == "nco": - nco_fix_dir = os.path.join(FIXLAM_NCO_BASEDIR, PREDEF_GRID_NAME) + nco_fix_dir = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) if not os.path.exists(nco_fix_dir): print_err_msg_exit(f''' The directory (nco_fix_dir) that should contain the pregenerated grid, @@ -1168,11 +1173,11 @@ def setup(): When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated grid files already exist in the directory - {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must @@ -1205,11 +1210,11 @@ def setup(): msg=f''' When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated orography files already exist in the directory - {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must @@ -1243,11 +1248,11 @@ def setup(): When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated surface climatology files already exist in the directory - {FIXLAM_NCO_BASEDIR}/{PREDEF_GRID_NAME} + {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} where - FIXLAM_NCO_BASEDIR = \"{FIXLAM_NCO_BASEDIR}\" + DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO From eb06e82748e9e6006235c5d677ece8c01a2edc3e Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 10 May 2022 03:59:39 -0400 Subject: [PATCH 167/203] Change output file name of run_post to meet NCO standards (#758) * change output file name * change variable name * update python script * remove duplicates * add a check for empty variables * move variable to common area * clean up unnecessary comments * update scripts * remove duplicate * update python scripts * fix user-staged dir path issue in python script --- scripts/exregional_run_ensgridvx.sh | 1 + scripts/exregional_run_ensgridvx_mean.sh | 1 + scripts/exregional_run_ensgridvx_prob.sh | 1 + scripts/exregional_run_enspointvx.sh | 1 + scripts/exregional_run_enspointvx_mean.sh | 1 + scripts/exregional_run_enspointvx_prob.sh | 1 + scripts/exregional_run_fcst.sh | 5 +- scripts/exregional_run_gridstatvx.sh | 1 + scripts/exregional_run_pointstatvx.sh | 1 + scripts/exregional_run_post.sh | 15 +- ush/config_defaults.sh | 12 ++ ush/config_defaults.yaml | 12 ++ ush/mrms_pull_topofhour.py | 160 +++++++++--------- ush/predef_grid_params.yaml | 45 ++++- ush/set_extrn_mdl_params.py | 33 +--- ush/set_predef_grid_params.py | 7 +- ush/setup.py | 30 +++- ush/setup.sh | 15 ++ .../parm/metplus/EnsembleStat_APCP01h.conf | 2 +- .../parm/metplus/EnsembleStat_APCP03h.conf | 6 +- .../parm/metplus/EnsembleStat_APCP06h.conf | 6 +- .../parm/metplus/EnsembleStat_APCP24h.conf | 6 +- .../parm/metplus/EnsembleStat_REFC.conf | 2 +- .../parm/metplus/EnsembleStat_RETOP.conf | 2 +- .../parm/metplus/EnsembleStat_conus_sfc.conf | 2 +- .../parm/metplus/EnsembleStat_upper_air.conf | 2 +- .../parm/metplus/GridStat_APCP01h.conf | 2 +- .../parm/metplus/GridStat_APCP03h.conf | 4 +- .../parm/metplus/GridStat_APCP06h.conf | 4 +- .../parm/metplus/GridStat_APCP24h.conf | 4 +- ush/templates/parm/metplus/GridStat_REFC.conf | 2 +- .../parm/metplus/GridStat_RETOP.conf | 2 +- .../parm/metplus/PointStat_conus_sfc.conf | 2 +- .../parm/metplus/PointStat_upper_air.conf | 2 +- ush/valid_param_vals.yaml | 3 +- 35 files changed, 241 insertions(+), 154 deletions(-) diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh index df1b91cd42..6d2cc1e403 100755 --- a/scripts/exregional_run_ensgridvx.sh +++ b/scripts/exregional_run_ensgridvx.sh @@ -133,6 +133,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME export NUM_ENS_MEMBERS export NUM_PAD export LOG_SUFFIX diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh index bd50e3cff0..9a18823eff 100755 --- a/scripts/exregional_run_ensgridvx_mean.sh +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -143,6 +143,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME export INPUT_BASE export LOG_SUFFIX diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh index 7345c1728b..8f45a53cd9 100755 --- a/scripts/exregional_run_ensgridvx_prob.sh +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -144,6 +144,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME export LOG_SUFFIX # diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh index 544f0f006d..ed1bd35817 100755 --- a/scripts/exregional_run_enspointvx.sh +++ b/scripts/exregional_run_enspointvx.sh @@ -135,6 +135,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME export NUM_ENS_MEMBERS ${METPLUS_PATH}/ush/run_metplus.py \ diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh index b65c9bd349..b9a0bc0c87 100755 --- a/scripts/exregional_run_enspointvx_mean.sh +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -137,6 +137,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh index b315faac34..4d2bf84642 100755 --- a/scripts/exregional_run_enspointvx_prob.sh +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -137,6 +137,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 331bb6ab49..2f768a6572 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -514,7 +514,6 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then yyyymmdd=${cdate:0:8} hh=${cdate:8:2} cyc=$hh - tmmark="tm00" fmn="00" if [ "${RUN_ENVIR}" = "nco" ]; then @@ -539,7 +538,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then post_mn=${post_time:10:2} post_mn_or_null="" post_fn_suffix="GrbF${fhr_d}" - post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${tmmark}.grib2" + post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${POST_OUTPUT_DOMAIN_NAME}.grib2" basetime=$( $DATE_UTIL --date "$yyyymmdd $hh" +%y%j%H%M ) symlink_suffix="_${basetime}f${fhr}${post_mn}" @@ -547,7 +546,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.t${cyc}z.${fid}${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.t${cyc}z.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${run_dir}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh index 63288c44ed..ae7f960fcf 100755 --- a/scripts/exregional_run_gridstatvx.sh +++ b/scripts/exregional_run_gridstatvx.sh @@ -150,6 +150,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh index 458b87749f..02a65ed2b9 100755 --- a/scripts/exregional_run_pointstatvx.sh +++ b/scripts/exregional_run_pointstatvx.sh @@ -140,6 +140,7 @@ export METPLUS_CONF export MET_CONFIG export MODEL export NET +export POST_OUTPUT_DOMAIN_NAME ${METPLUS_PATH}/ush/run_metplus.py \ -c ${METPLUS_CONF}/common.conf \ diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index c7aafd3997..cb8c3d5d5e 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -172,17 +172,6 @@ cyc=$hh # #----------------------------------------------------------------------- # -# The tmmark is a reference value used in real-time, DA-enabled NCEP models. -# It represents the delay between the onset of the DA cycle and the free -# forecast. With no DA in the SRW App at the moment, it is hard-wired to -# tm00 for now. -# -#----------------------------------------------------------------------- -# -tmmark="tm00" -# -#----------------------------------------------------------------------- -# # Create the namelist file (itag) containing arguments to pass to the post- # processor's executable. # @@ -300,7 +289,7 @@ if [ "${post_mn}" != "00" ]; then fi post_fn_suffix="GrbF${post_fhr}${dot_post_mn_or_null}" -post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${tmmark}.grib2" +post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${POST_OUTPUT_DOMAIN_NAME}.grib2" # # For convenience, change location to postprd_dir (where the final output # from UPP will be located). Then loop through the two files that UPP @@ -314,7 +303,7 @@ fids=( "prslev" "natlev" ) for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.t${cyc}z.${fid}${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.t${cyc}z.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${tmp_dir}/${post_orig_fn} ${post_renamed_fn} create_symlink_to_file target="${post_renamed_fn}" \ symlink="${FID}${symlink_suffix}" \ diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 7cce8c1a69..d304d203a4 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -411,6 +411,18 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # #----------------------------------------------------------------------- # +# Set output file name. Definitions: +# +# POST_OUTPUT_DOMAIN_NAME: +# Domain name used in naming the output files of run_post by UPP or inline post. +# Output file name: $NET.tHHz.[var_name].f###.$POST_OUTPUT_DOMAIN_NAME.grib2 +# +#----------------------------------------------------------------------- +# +POST_OUTPUT_DOMAIN_NAME="" +# +#----------------------------------------------------------------------- +# # Set forecast parameters. Definitions: # # DATE_FIRST_CYCL: diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 6dbd311102..56b89bb117 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -410,6 +410,18 @@ WFLOW_LAUNCH_LOG_FN: "log.launch_FV3LAM_wflow" # #----------------------------------------------------------------------- # +# Set output file name. Definitions: +# +# POST_OUTPUT_DOMAIN_NAME: +# Domain name used in naming the output files of run_post by UPP or inline post. +# Output file name: $NET.tHHz.[var_name].f###.$POST_OUTPUT_DOMAIN_NAME.grib2 +# +#----------------------------------------------------------------------- +# +POST_OUTPUT_DOMAIN_NAME: "" +# +#----------------------------------------------------------------------- +# # Set forecast parameters. Definitions: # # DATE_FIRST_CYCL: diff --git a/ush/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py index bd98b28058..3744242019 100644 --- a/ush/mrms_pull_topofhour.py +++ b/ush/mrms_pull_topofhour.py @@ -3,80 +3,86 @@ import re, csv, glob import bisect import numpy as np - -# Copy and unzip MRMS files that are closest to top of hour -# Done every hour on a 20-minute lag - -# Include option to define valid time on command line -# Used to backfill verification -#try: -valid_time = str(sys.argv[1]) - -YYYY = int(valid_time[0:4]) -MM = int(valid_time[4:6]) -DD = int(valid_time[6:8]) -HH = int(valid_time[8:19]) - -valid = datetime.datetime(YYYY,MM,DD,HH,0,0) - -#except IndexError: -# valid_time = None - -# Default to current hour if not defined on command line -#if valid_time is None: -# now = datetime.datetime.utcnow() -# YYYY = int(now.strftime('%Y')) -# MM = int(now.strftime('%m')) -# DD = int(now.strftime('%d')) -# HH = int(now.strftime('%H')) - -# valid = datetime.datetime(YYYY,MM,DD,HH,0,0) -# valid_time = valid.strftime('%Y%m%d%H') - -print('Pulling '+valid_time+' MRMS data') - -# Set up working directory -DATA_HEAD = str(sys.argv[2]) -MRMS_PROD_DIR = str(sys.argv[3]) -MRMS_PRODUCT = str(sys.argv[4]) -level = str(sys.argv[5]) - -VALID_DIR = os.path.join(DATA_HEAD,valid.strftime('%Y%m%d')) -if not os.path.exists(VALID_DIR): - os.makedirs(VALID_DIR) -os.chdir(DATA_HEAD) - -# Sort list of files for each MRMS product -print(valid.strftime('%Y%m%d')) -if valid.strftime('%Y%m%d') < '20200303': - search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' -elif valid.strftime('%Y%m%d') >= '20200303': - search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' -file_list = [f for f in glob.glob(search_path)] -time_list = [file_list[x][-24:-9] for x in range(len(file_list))] -int_list = [int(time_list[x][0:8]+time_list[x][9:15]) for x in range(len(time_list))] -int_list.sort() -datetime_list = [datetime.datetime.strptime(str(x),"%Y%m%d%H%M%S") for x in int_list] - -# Find the MRMS file closest to the valid time -i = bisect.bisect_left(datetime_list,valid) -closest_timestamp = min(datetime_list[max(0, i-1): i+2], key=lambda date: abs(valid - date)) - -# Check to make sure closest file is within +/- 15 mins of top of the hour -# Copy and rename the file for future ease -difference = abs(closest_timestamp - valid) -if difference.total_seconds() <= 900: - filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz' - filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz' - - if valid.strftime('%Y%m%d') < '20200303': - print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) - - os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) - os.system('gunzip '+VALID_DIR+'/'+filename2) - elif valid.strftime('%Y%m%d') >= '20200303': - print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) - - os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) - os.system('gunzip '+VALID_DIR+'/'+filename2) - +import unittest + +if __name__ == '__main__': + # Copy and unzip MRMS files that are closest to top of hour + # Done every hour on a 20-minute lag + + # Include option to define valid time on command line + # Used to backfill verification + #try: + valid_time = str(sys.argv[1]) + + YYYY = int(valid_time[0:4]) + MM = int(valid_time[4:6]) + DD = int(valid_time[6:8]) + HH = int(valid_time[8:19]) + + valid = datetime.datetime(YYYY,MM,DD,HH,0,0) + + #except IndexError: + # valid_time = None + + # Default to current hour if not defined on command line + #if valid_time is None: + # now = datetime.datetime.utcnow() + # YYYY = int(now.strftime('%Y')) + # MM = int(now.strftime('%m')) + # DD = int(now.strftime('%d')) + # HH = int(now.strftime('%H')) + + # valid = datetime.datetime(YYYY,MM,DD,HH,0,0) + # valid_time = valid.strftime('%Y%m%d%H') + + print('Pulling '+valid_time+' MRMS data') + + # Set up working directory + DATA_HEAD = str(sys.argv[2]) + MRMS_PROD_DIR = str(sys.argv[3]) + MRMS_PRODUCT = str(sys.argv[4]) + level = str(sys.argv[5]) + + VALID_DIR = os.path.join(DATA_HEAD,valid.strftime('%Y%m%d')) + if not os.path.exists(VALID_DIR): + os.makedirs(VALID_DIR) + os.chdir(DATA_HEAD) + + # Sort list of files for each MRMS product + print(valid.strftime('%Y%m%d')) + if valid.strftime('%Y%m%d') < '20200303': + search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' + elif valid.strftime('%Y%m%d') >= '20200303': + search_path = MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+MRMS_PRODUCT+'*.gz' + file_list = [f for f in glob.glob(search_path)] + time_list = [file_list[x][-24:-9] for x in range(len(file_list))] + int_list = [int(time_list[x][0:8]+time_list[x][9:15]) for x in range(len(time_list))] + int_list.sort() + datetime_list = [datetime.datetime.strptime(str(x),"%Y%m%d%H%M%S") for x in int_list] + + # Find the MRMS file closest to the valid time + i = bisect.bisect_left(datetime_list,valid) + closest_timestamp = min(datetime_list[max(0, i-1): i+2], key=lambda date: abs(valid - date)) + + # Check to make sure closest file is within +/- 15 mins of top of the hour + # Copy and rename the file for future ease + difference = abs(closest_timestamp - valid) + if difference.total_seconds() <= 900: + filename1 = MRMS_PRODUCT+level+closest_timestamp.strftime('%Y%m%d-%H%M%S')+'.grib2.gz' + filename2 = MRMS_PRODUCT+level+valid.strftime('%Y%m%d-%H')+'0000.grib2.gz' + + if valid.strftime('%Y%m%d') < '20200303': + print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + + os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/dcom/us007003/ldmdata/obs/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + os.system('gunzip '+VALID_DIR+'/'+filename2) + elif valid.strftime('%Y%m%d') >= '20200303': + print('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + + os.system('cp '+MRMS_PROD_DIR+'/'+valid.strftime('%Y%m%d')+'/upperair/mrms/conus/'+MRMS_PRODUCT+'/'+filename1+' '+VALID_DIR+'/'+filename2) + os.system('gunzip '+VALID_DIR+'/'+filename2) + +#dummy unittest +class Testing(unittest.TestCase): + def test_mrms_pull_topfhour(self): + pass diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index 39701ca62d..330a9a6472 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -356,7 +356,7 @@ WRTCMP_stdlat1: 39.77 WRTCMP_stdlat2: 39.77 WRTCMP_nx: 197 - WRTCMP_ny: 195 + WRTCMP_ny: 197 WRTCMP_lon_lwr_left: -89.47120417 WRTCMP_lat_lwr_left: 37.07809642 WRTCMP_dx: 3000.0 @@ -440,6 +440,49 @@ # #----------------------------------------------------------------------- # +# The WoFS domain with ~3km cells. +# +# Note: +# The WoFS domain will generate a 301 x 301 output grid (WRITE COMPONENT) and +# will eventually be movable (ESGgrid_LON_CTR/ESGgrid_LAT_CTR). A python script +# python_utils/fv3write_parms_lambert will be useful to determine +# WRTCMP_lon_lwr_left and WRTCMP_lat_lwr_left locations (only for Lambert map +# projection currently) of the quilting output when the domain location is +# moved. Later, it should be integrated into the workflow. +# +#----------------------------------------------------------------------- +# +"WoFS_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -97.5 + ESGgrid_LAT_CTR: 38.5 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 361 + ESGgrid_NY: 361 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 20 + LAYOUT_X: 18 + LAYOUT_Y: 12 + BLOCKSIZE: 30 + # if QUILTING = True + WRTCMP_write_groups: "1" + WRTCMP_write_tasks_per_group: 12 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -97.5 + WRTCMP_cen_lat: 38.5 + WRTCMP_stdlat1: 38.5 + WRTCMP_stdlat2: 38.5 + WRTCMP_nx: 301 + WRTCMP_ny: 301 + WRTCMP_lon_lwr_left: -102.3802487 + WRTCMP_lat_lwr_left: 34.3407918 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# # A CONUS domain of GFDLgrid type with ~25km cells. # # Note: diff --git a/ush/set_extrn_mdl_params.py b/ush/set_extrn_mdl_params.py index 1eea1f2bab..f4fffc5907 100644 --- a/ush/set_extrn_mdl_params.py +++ b/ush/set_extrn_mdl_params.py @@ -16,24 +16,8 @@ def set_extrn_mdl_params(): #import all env variables import_vars() - global EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS, EXTRN_MDL_LBCS_OFFSET_HRS + global EXTRN_MDL_LBCS_OFFSET_HRS - # - #----------------------------------------------------------------------- - # - # Use known locations or COMIN as default, depending on RUN_ENVIR - # - #----------------------------------------------------------------------- - # - if RUN_ENVIR == "nco": - EXTRN_MDL_SYSBASEDIR_ICS=EXTRN_MDL_SYSBASEDIR_ICS or COMIN - EXTRN_MDL_SYSBASEDIR_LBCS=EXTRN_MDL_SYSBASEDIR_LBCS or COMIN - else: - ics_dir=EXTRN_MDL_NAME_ICS - lbcs_dir=EXTRN_MDL_NAME_LBCS - EXTRN_MDL_SYSBASEDIR_ICS=EXTRN_MDL_SYSBASEDIR_ICS or ics_dir - EXTRN_MDL_SYSBASEDIR_LBCS=EXTRN_MDL_SYSBASEDIR_LBCS or lbcs_dir - # #----------------------------------------------------------------------- # @@ -49,7 +33,7 @@ def set_extrn_mdl_params(): EXTRN_MDL_LBCS_OFFSET_HRS=EXTRN_MDL_LBCS_OFFSET_HRS or "0" # export values we set above - env_vars = ["EXTRN_MDL_SYSBASEDIR_ICS", "EXTRN_MDL_SYSBASEDIR_LBCS", "EXTRN_MDL_LBCS_OFFSET_HRS"] + env_vars = ["EXTRN_MDL_LBCS_OFFSET_HRS"] export_vars(env_vars=env_vars) # #----------------------------------------------------------------------- @@ -64,16 +48,9 @@ def set_extrn_mdl_params(): class Testing(unittest.TestCase): def test_extrn_mdl_params(self): set_extrn_mdl_params() - EXTRN_MDL_SYSBASEDIR_ICS = get_env_var("EXTRN_MDL_SYSBASEDIR_ICS") - COMINgfs = get_env_var("COMINgfs") - self.assertEqual(EXTRN_MDL_SYSBASEDIR_ICS,COMINgfs) + EXTRN_MDL_LBCS_OFFSET_HRS = get_env_var("EXTRN_MDL_LBCS_OFFSET_HRS") + self.assertEqual(EXTRN_MDL_LBCS_OFFSET_HRS,3) def setUp(self): - set_env_var("MACHINE","HERA") - set_env_var("RUN_ENVIR","nco") - set_env_var("EXTRN_MDL_NAME_ICS","FV3GFS") - set_env_var("EXTRN_MDL_NAME_LBCS","FV3GFS") - set_env_var("EXTRN_MDL_SYSBASEDIR_ICS",None) - set_env_var("EXTRN_MDL_SYSBASEDIR_LBCS",None) + set_env_var("EXTRN_MDL_NAME_LBCS","RAP") set_env_var("EXTRN_MDL_LBCS_OFFSET_HRS",None) - set_env_var("COMINgfs","/base/path/of/directory/containing/gfs/input/files") diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index e8f4e94ab3..2838517150 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -23,6 +23,11 @@ def set_predef_grid_params(): params_dict = load_config_file("predef_grid_params.yaml") params_dict = params_dict[PREDEF_GRID_NAME] + # if QUILTING = False, skip variables that start with "WRTCMP_" + if not QUILTING: + params_dict = {k: v for k,v in params_dict.items() \ + if not k.startswith("WRTCMP_") } + # take care of special vars special_vars = ['DT_ATMOS', 'LAYOUT_X', 'LAYOUT_Y', 'BLOCKSIZE'] for var in special_vars: @@ -55,5 +60,5 @@ def setUp(self): set_env_var('LAYOUT_X',18) set_env_var('LAYOUT_Y',36) set_env_var('BLOCKSIZE',28) - set_env_var('QUILTING',True) + set_env_var('QUILTING',False) diff --git a/ush/setup.py b/ush/setup.py index acf46f1cdf..a7621671c4 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -821,10 +821,13 @@ def setup(): # # COMOUT_BASEDIR is not used by the workflow in community mode. # + # POST_OUTPUT_DOMAIN_NAME: + # The PREDEF_GRID_NAME is set by default. + # #----------------------------------------------------------------------- # global LOGDIR, FIXam, FIXclim, FIXLAM, CYCLE_BASEDIR, \ - COMROOT, COMOUT_BASEDIR + COMROOT, COMOUT_BASEDIR, POST_OUTPUT_DOMAIN_NAME LOGDIR = os.path.join(EXPTDIR, "log") @@ -845,6 +848,20 @@ def setup(): CYCLE_BASEDIR=EXPTDIR COMROOT="" COMOUT_BASEDIR="" + + if POST_OUTPUT_DOMAIN_NAME is None: + if PREDEF_GRID_NAME is None: + print_err_msg_exit(f''' + The domain name used in naming the run_post output files + (POST_OUTPUT_DOMAIN_NAME) has not been set: + POST_OUTPUT_DOMAIN_NAME = \"${POST_OUTPUT_DOMAIN_NAME}\" + If this experiment is not using a predefined grid (i.e. if + PREDEF_GRID_NAME is set to a null string), POST_OUTPUT_DOMAIN_NAME + must be set in the configuration file (\"${EXPT_CONFIG_FN}\"). ''') + + POST_OUTPUT_DOMAIN_NAME = PREDEF_GRID_NAME + + POST_OUTPUT_DOMAIN_NAME = lowercase(POST_OUTPUT_DOMAIN_NAME) # #----------------------------------------------------------------------- # @@ -1040,19 +1057,22 @@ def setup(): #----------------------------------------------------------------------- # if USE_USER_STAGED_EXTRN_FILES: - - if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS): # Check for the base directory up to the first templated field. idx = EXTRN_MDL_SOURCE_BASEDIR_ICS.find("$") + if idx == -1: + idx=len(EXTRN_MDL_SOURCE_BASEDIR_ICS) + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]): print_err_msg_exit(f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''') - if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS): idx = EXTRN_MDL_SOURCE_BASEDIR_LBCS.find("$") - if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): + if idx == -1: + idx=len(EXTRN_MDL_SOURCE_BASEDIR_LBCS) + + if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): print_err_msg_exit(f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: diff --git a/ush/setup.sh b/ush/setup.sh index 67b6776926..a5e0f15fda 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1155,6 +1155,9 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # # COMOUT_BASEDIR is not used by the workflow in community mode. # +# POST_OUTPUT_DOMAIN_NAME: +# The PREDEF_GRID_NAME is set by default. +# #----------------------------------------------------------------------- # LOGDIR="${EXPTDIR}/log" @@ -1174,6 +1177,18 @@ else COMROOT="" COMOUT_BASEDIR="" fi + +POST_OUTPUT_DOMAIN_NAME="${POST_OUTPUT_DOMAIN_NAME:-${PREDEF_GRID_NAME}}" +if [ -z "${POST_OUTPUT_DOMAIN_NAME}" ]; then + print_err_msg_exit "\ +The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) +has not been set: + POST_OUTPUT_DOMAIN_NAME = \"${POST_OUTPUT_DOMAIN_NAME}\" +If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME +is set to a null string), POST_OUTPUT_DOMAIN_NAME must be set in the SRW +App's configuration file (\"${EXPT_CONFIG_FN}\")." +fi +POST_OUTPUT_DOMAIN_NAME=$(echo_lowercase ${POST_OUTPUT_DOMAIN_NAME}) # #----------------------------------------------------------------------- # diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf index 23f6adae9d..33599d9f79 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf @@ -221,7 +221,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf index d67380ad80..2691dc2dca 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf @@ -256,8 +256,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -266,7 +266,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a03h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf index ae2f61f449..8527e23635 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf @@ -257,8 +257,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -267,7 +267,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a06h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf index 0b1d175af0..c3ed6a3a99 100644 --- a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf +++ b/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf @@ -257,8 +257,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -267,7 +267,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a24h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_REFC.conf b/ush/templates/parm/metplus/EnsembleStat_REFC.conf index 989341a550..123af0f0a2 100644 --- a/ush/templates/parm/metplus/EnsembleStat_REFC.conf +++ b/ush/templates/parm/metplus/EnsembleStat_REFC.conf @@ -220,7 +220,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/REFC # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf index b0c235a2e8..4fcbc9bcf2 100644 --- a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf +++ b/ush/templates/parm/metplus/EnsembleStat_RETOP.conf @@ -223,7 +223,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/RETOP # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index e1d95040c0..d660664228 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -325,7 +325,7 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # or a single line, - filename wildcard characters may be used, ? or *. FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf index 6251fbac13..072926a3b1 100644 --- a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -394,7 +394,7 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # or a single line, - filename wildcard characters may be used, ? or *. FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 + mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/ush/templates/parm/metplus/GridStat_APCP01h.conf b/ush/templates/parm/metplus/GridStat_APCP01h.conf index 240462dacd..2c6aac65aa 100644 --- a/ush/templates/parm/metplus/GridStat_APCP01h.conf +++ b/ush/templates/parm/metplus/GridStat_APCP01h.conf @@ -253,7 +253,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 diff --git a/ush/templates/parm/metplus/GridStat_APCP03h.conf b/ush/templates/parm/metplus/GridStat_APCP03h.conf index f6d6290215..c017806171 100644 --- a/ush/templates/parm/metplus/GridStat_APCP03h.conf +++ b/ush/templates/parm/metplus/GridStat_APCP03h.conf @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,7 +291,7 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat diff --git a/ush/templates/parm/metplus/GridStat_APCP06h.conf b/ush/templates/parm/metplus/GridStat_APCP06h.conf index 7c8d69d1a3..0e4d66531e 100644 --- a/ush/templates/parm/metplus/GridStat_APCP06h.conf +++ b/ush/templates/parm/metplus/GridStat_APCP06h.conf @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,7 +291,7 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat diff --git a/ush/templates/parm/metplus/GridStat_APCP24h.conf b/ush/templates/parm/metplus/GridStat_APCP24h.conf index bcccfe8e18..7ae700cf31 100644 --- a/ush/templates/parm/metplus/GridStat_APCP24h.conf +++ b/ush/templates/parm/metplus/GridStat_APCP24h.conf @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,7 +291,7 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat diff --git a/ush/templates/parm/metplus/GridStat_REFC.conf b/ush/templates/parm/metplus/GridStat_REFC.conf index 7eade0cb1d..027723470e 100644 --- a/ush/templates/parm/metplus/GridStat_REFC.conf +++ b/ush/templates/parm/metplus/GridStat_REFC.conf @@ -263,7 +263,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/REFC [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 diff --git a/ush/templates/parm/metplus/GridStat_RETOP.conf b/ush/templates/parm/metplus/GridStat_RETOP.conf index 4ef239d362..0b00292112 100644 --- a/ush/templates/parm/metplus/GridStat_RETOP.conf +++ b/ush/templates/parm/metplus/GridStat_RETOP.conf @@ -263,7 +263,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/RETOP [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index f6e231699c..b3a2755a58 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -273,7 +273,7 @@ PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index 1f24cbab46..6110122c5d 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -273,7 +273,7 @@ PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslevf{lead?fmt=%HHH}.tm{init?fmt=%H}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index f9a00e95b4..efadce6088 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -30,7 +30,8 @@ valid_vals_PREDEF_GRID_NAME: [ "GSD_HRRR_AK_50km", "RRFS_NA_13km", "RRFS_NA_3km", -"SUBCONUS_Ind_3km" +"SUBCONUS_Ind_3km", +"WoFS_3km" ] valid_vals_CCPP_PHYS_SUITE: [ "FV3_GFS_2017_gfdlmp", From c9e1fbf0fc0a3f58e7b9e3f4aff7d9490438abd6 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 10 May 2022 15:20:49 -0400 Subject: [PATCH 168/203] Add POST_OUTPUT_DOMAIN_NAME to WE2E tests for new grids (#763) * Add new var to we2e tests for new grids * rename we2e tests for custom grid * remove unnecessary $ --- .../{config.new_ESGgrid.sh => config.custom_ESGgrid.sh} | 2 ++ .../{config.new_GFDLgrid.sh => config.custom_GFDLgrid.sh} | 2 ++ ...Lgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh} | 2 ++ ...DLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh} | 2 ++ ush/setup.py | 4 ++-- 5 files changed, 10 insertions(+), 2 deletions(-) rename tests/WE2E/test_configs/wflow_features/{config.new_ESGgrid.sh => config.custom_ESGgrid.sh} (96%) rename tests/WE2E/test_configs/wflow_features/{config.new_GFDLgrid.sh => config.custom_GFDLgrid.sh} (98%) rename tests/WE2E/test_configs/wflow_features/{config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh => config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh} (97%) rename tests/WE2E/test_configs/wflow_features/{config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh => config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh} (97%) diff --git a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh similarity index 96% rename from tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh index fc34f4a0c3..bb8d2bc0cf 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh @@ -44,6 +44,8 @@ LAYOUT_X="8" LAYOUT_Y="12" BLOCKSIZE="13" +POST_OUTPUT_DOMAIN_NAME="custom_ESGgrid" + QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh similarity index 98% rename from tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh index d27148de2a..66604a5491 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh @@ -76,6 +76,8 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" + QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh similarity index 97% rename from tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh index aef2636f2e..4c4f0d5c75 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh @@ -63,6 +63,8 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" + QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh similarity index 97% rename from tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh index 59ca0925e7..3e067ee3d2 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh @@ -63,6 +63,8 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" + QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/ush/setup.py b/ush/setup.py index a7621671c4..2e6cb912c4 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -854,10 +854,10 @@ def setup(): print_err_msg_exit(f''' The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) has not been set: - POST_OUTPUT_DOMAIN_NAME = \"${POST_OUTPUT_DOMAIN_NAME}\" + POST_OUTPUT_DOMAIN_NAME = \"{POST_OUTPUT_DOMAIN_NAME}\" If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME is set to a null string), POST_OUTPUT_DOMAIN_NAME - must be set in the configuration file (\"${EXPT_CONFIG_FN}\"). ''') + must be set in the configuration file (\"{EXPT_CONFIG_FN}\"). ''') POST_OUTPUT_DOMAIN_NAME = PREDEF_GRID_NAME From d1fa82693a899508d859bdfd6a2d42faf9703485 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 12 May 2022 19:54:41 -0600 Subject: [PATCH 169/203] Modifications to `CODEOWNERS` file (#757) * Add @gspetro-NOAA, @natalie-perlin, and @EdwardSnyder-NOAA to CODEOWNERS so they are notified of all PRs and can review them. * Remove duplicates in CODEOWNERS; remove users who will no longer be working with the repo. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 154c5ba16d..2aa60c048c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # These owners will be the default owners for everything in the repo. #* @defunkt -* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @BenjaminBlake-NOAA @RatkoVasic-NOAA @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @jwolff-ncar @dmwright526 +* @mkavulich @gsketefian @JeffBeck-NOAA @RatkoVasic-NOAA @BenjaminBlake-NOAA @ywangwof @chan-hoo @panll @christinaholtNOAA @christopherwharrop-noaa @danielabdi-noaa @mark-a-potts @jkbk2004 @willmayfield @dmwright526 @gspetro-NOAA @natalie-perlin @EdwardSnyder-NOAA # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners From 85ef19b444ff8234c5fe8c00f38e0f1f9bd91dfa Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Mon, 16 May 2022 12:16:27 -0600 Subject: [PATCH 170/203] Adding a python utility for summarizing compute. (#769) Adds a utility that summarizes Rocoto database computational usage information. --- tests/WE2E/create_WE2E_resource_summary.py | 187 +++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 tests/WE2E/create_WE2E_resource_summary.py diff --git a/tests/WE2E/create_WE2E_resource_summary.py b/tests/WE2E/create_WE2E_resource_summary.py new file mode 100644 index 0000000000..5095a9fe69 --- /dev/null +++ b/tests/WE2E/create_WE2E_resource_summary.py @@ -0,0 +1,187 @@ +''' +Generate a summary of resources used for the WE2E test suite. + +Examples: + + To print usage + + python create_WE2E_resource_summary.py + python create_WE2E_resource_summary.py -h + + To print a report for all the experiments in an experiment directory + + python create_WE2E_resource_summary.py -e /path/to/expt_dir + + To print a report for all the grid_* and nco_* experiments. + + python create_WE2E_resource_summary.py -e /path/to/expt_dir \ + -n 'grid*' 'nco*' + + To compute a total estimated cost for all experiments on instances that are + $0.15 per core hour. + + python create_WE2E_resource_summary.py -e /path/to/expt_dir -c $0.15 + +Information about the output summary. + + - The core hours are an underestimate in many cases. + - Multiple tries are not captured. + - The use of a portion of a node or instance is not known. If the whole node + is used, but isn't reflected in the core count, the cores are not counted. + Partition information is not stored in the database, so mapping to a given + node type becomes ambiguous. + + For example, jobs that request 4 nodes with 2 processors per node with an + --exclusive flag will underestimate the total core hour usage by a factor + of 20 when using a 40 processor node. + + - When computing cost per job, it will also provide an underestimate for the + reasons listed above. + - Only one cost will be applied across all jobs. Rocoto jobs do not store + partition information in the job table, so was not included as an option here. + +''' + +import argparse +import glob +import os +import sys +import sqlite3 + +REPORT_WIDTH = 110 + +def parse_args(argv): + + + ''' + Function maintains the arguments accepted by this script. Please see + Python's argparse documenation for more information about settings of each + argument. + ''' + + parser = argparse.ArgumentParser( + description="Generate a usage report for a set of SRW experiments." + ) + + parser.add_argument( + '-e', '--expt_path', + help='The path to the directory containing the experiment \ + directories', + ) + parser.add_argument( + '-n', '--expt_names', + default=['*'], + help='A list of experiments to generate the report for. Wildcards \ + accepted by glob.glob may be used. If not provided, a report will be \ + generated for all experiments in the expt_path that have a Rocoto \ + database', + nargs='*', + ) + + # Optional + parser.add_argument( + '-c', '--cost_per_core_hour', + help='Provide the cost per core hour for the instance type used. \ + Only supports homogenous clusters.', + type=float, + ) + + return parser.parse_args(argv) + +def get_workflow_info(db_path): + + ''' Given the path to a Rocoto database, return the total number of tasks, + core hours and wall time for the workflow. ''' + + con = sqlite3.connect(db_path) + cur = con.cursor() + + # jobs schema is: + # (id INTEGER PRIMARY KEY, jobid VARCHAR(64), taskname VARCHAR(64), cycle + # DATETIME, cores INTEGER, state VARCHAR(64), native_state VARCHAR[64], + # exit_status INTEGER, tries INTEGER, nunknowns INTEGER, duration REAL) + # + # an example: + # 5|66993580|make_sfc_climo|1597017600|48|SUCCEEDED|COMPLETED|0|1|0|83.0 + try: + cur.execute('SELECT cores, duration from jobs') + except sqlite3.OperationalError: + return 0, 0, 0 + + workflow_info = cur.fetchall() + + core_hours = 0 + wall_time = 0 + ntasks = 0 + for cores, duration in workflow_info: + core_hours += cores * duration / 3600 + wall_time += duration / 60 + ntasks += 1 + + return ntasks, core_hours, wall_time + + +def fetch_expt_summaries(expts): + + ''' Get the important information from the database of each experiment, and + return a list, sorted by experiment name. ''' + + summaries = [] + for expt in expts: + test_name = expt.split('/')[-1] + db_path = os.path.join(expt, 'FV3LAM_wflow.db') + if not os.path.exists(db_path): + print(f'No FV3LAM_wflow.db exists for expt: {test_name}') + continue + ntasks, core_hours, wall_time = get_workflow_info(db_path) + summaries.append((test_name, ntasks, core_hours, wall_time)) + + return sorted(summaries) + +def generate_report(argv): + + ''' Given user arguments, print a summary of the requested experiments' + usage information, including cost (if requested). ''' + + cla = parse_args(argv) + + experiments = [] + for expt in cla.expt_names: + experiments.extend(glob.glob( + os.path.join(cla.expt_path, expt) + )) + + header = f'{" "*60} Core Hours | Run Time (mins)' + if cla.cost_per_core_hour: + header = f'{header} | Est. Cost ($) ' + + print('-'*REPORT_WIDTH) + print('-'*REPORT_WIDTH) + print(header) + print('-'*REPORT_WIDTH) + + total_ch = 0 + total_cost = 0 + for name, ntasks, ch, wt in fetch_expt_summaries(experiments): + line = f'{name[:60]:<60s} {ch:^12.2f} {wt:^20.1f}' + if cla.cost_per_core_hour: + cost = ch * cla.cost_per_core_hour + line = f'{line} ${cost:<.2f}' + total_cost += cost + total_ch += ch + print(line) + + print('-'*REPORT_WIDTH) + print(f'TOTAL CORE HOURS: {total_ch:6.2f}') + if cla.cost_per_core_hour: + print(f'TOTAL COST: ${cla.cost_per_core_hour * total_ch:6.2f}') + + print('*'*REPORT_WIDTH) + print('WARNING: This data reflects only the job information from the last', + 'logged try. It does not account for the use \n of an entire node, only', + 'the actual cores requested. It may provide an underestimate of true compute usage.') + print('*'*REPORT_WIDTH) + + +if __name__ == "__main__": + generate_report(sys.argv[1:]) From 6f50e046490b360279124849e9a74ef2a8a70e06 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Mon, 16 May 2022 12:41:16 -0600 Subject: [PATCH 171/203] Add github actions for python unittests. (#747) * Add github actions for python unittests. * Include all python script in ush * Skip defining QUILTING params when it is set to False * Update py_workflow * Update unittest for set_extrn_mdl_params. * Updates from develop. Co-authored-by: Daniel Shawul --- .github/workflows/python_unittests.yaml | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/python_unittests.yaml diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml new file mode 100644 index 0000000000..65688fd387 --- /dev/null +++ b/.github/workflows/python_unittests.yaml @@ -0,0 +1,27 @@ +name: Python unittests +on: [push, pull_request] +jobs: + + python_unittests: + name: Python unittests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Install dependencies + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install python3 python3-pip netcdf-bin + sudo pip3 install pyyaml jinja2 f90nml + sudo pip3 install numpy matplotlib basemap + + # Run python unittests + - name: Run python unittests + run: | + cd ush + python3 -m unittest -b python_utils/test_python_utils.py + python3 -m unittest -b *.py + From 3490cbb7dde80ea88d53a4ba9257dc28dd545bf7 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 17 May 2022 13:22:03 -0400 Subject: [PATCH 172/203] Update sample script for NCO mode (#771) * update config.nco.sh * Add comment --- ush/config.nco.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/ush/config.nco.sh b/ush/config.nco.sh index 2e136c773c..556aa65218 100644 --- a/ush/config.nco.sh +++ b/ush/config.nco.sh @@ -8,6 +8,9 @@ VERBOSE="TRUE" RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" +USE_CRON_TO_RELAUNCH="TRUE" +CRON_RELAUNCH_INTVL_MNTS="3" + PREDEF_GRID_NAME="RRFS_CONUS_25km" QUILTING="TRUE" @@ -26,16 +29,23 @@ EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" FV3GFS_FILE_FMT_LBCS="grib2" +WTIME_RUN_FCST="01:00:00" + WRITE_DOPOST="TRUE" # -# The following must be modified for different platforms and users. +# Output directory: {NET}/{model_ver}/{RUN}.YYYYMMDD/ +# Output file name: {NET}.tHHz.[var_name].f###.{POST_OUTPUT_DOMAIN_NAME}.grib2 # +POST_OUTPUT_DOMAIN_NAME="conus_25km" NET="rrfs" model_ver="v1.0" RUN="rrfs_test" -COMIN="/scratch1/NCEPDEV/rstprod/com/gfs/prod" # Path to directory containing files from the external model. -FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" # Path to directory containing the pregenerated grid, orography, and surface climatology "fixed" files to use for the experiment. -STMP="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/01_OUT_DATA/stmp" # Path to directory STMP that mostly contains input files. -PTMP="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/01_OUT_DATA/ptmp" # Path to directory PTMP in which the experiment's output files will be placed. +# +# The following must be modified for different platforms and users. +# +COMIN="/scratch1/NCEPDEV/rstprod/com/gfs/prod" # Path to directory containing files from the external model. +DOMAIN_PREGEN_BASEDIR="/scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen" # Path to directory containing the pregenerated grid, orography, and surface climatology "fixed" files to use for the experiment. +STMP="/path/to/stmp/directory" # Path to directory STMP that mostly contains input files. +PTMP="/path/to/ptmp/directory" # Path to directory PTMP in which the experiment's output files will be placed. From 4946ed9b1a0c55dda57dfe9293a491a61d8798c0 Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Thu, 19 May 2022 08:27:31 -0400 Subject: [PATCH 173/203] Feature/noaacloud (#767) * updates for noaacloud * working version * fixes for noaacloud * added extra modules for post --- modulefiles/tasks/noaacloud/get_extrn_ics.local | 7 +++++++ modulefiles/tasks/noaacloud/get_extrn_lbcs.local | 6 ++++++ modulefiles/tasks/noaacloud/make_grid.local | 12 ++++++++++++ modulefiles/tasks/noaacloud/make_ics.local | 14 ++++++++++++++ modulefiles/tasks/noaacloud/make_lbcs.local | 14 ++++++++++++++ modulefiles/tasks/noaacloud/make_orog.local | 12 ++++++++++++ modulefiles/tasks/noaacloud/make_sfc_climo.local | 12 ++++++++++++ .../tasks/noaacloud/miniconda_regional_workflow | 5 +++++ modulefiles/tasks/noaacloud/run_fcst.local | 14 ++++++++++++++ modulefiles/tasks/noaacloud/run_post.local | 14 ++++++++++++++ ush/machine/noaacloud.sh | 6 +++++- 11 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 modulefiles/tasks/noaacloud/get_extrn_ics.local create mode 100644 modulefiles/tasks/noaacloud/get_extrn_lbcs.local create mode 100644 modulefiles/tasks/noaacloud/make_grid.local create mode 100644 modulefiles/tasks/noaacloud/make_ics.local create mode 100644 modulefiles/tasks/noaacloud/make_lbcs.local create mode 100644 modulefiles/tasks/noaacloud/make_orog.local create mode 100644 modulefiles/tasks/noaacloud/make_sfc_climo.local create mode 100644 modulefiles/tasks/noaacloud/miniconda_regional_workflow create mode 100644 modulefiles/tasks/noaacloud/run_fcst.local create mode 100644 modulefiles/tasks/noaacloud/run_post.local diff --git a/modulefiles/tasks/noaacloud/get_extrn_ics.local b/modulefiles/tasks/noaacloud/get_extrn_ics.local new file mode 100644 index 0000000000..19f582feda --- /dev/null +++ b/modulefiles/tasks/noaacloud/get_extrn_ics.local @@ -0,0 +1,7 @@ +#%Module + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow + diff --git a/modulefiles/tasks/noaacloud/get_extrn_lbcs.local b/modulefiles/tasks/noaacloud/get_extrn_lbcs.local new file mode 100644 index 0000000000..6b0d974687 --- /dev/null +++ b/modulefiles/tasks/noaacloud/get_extrn_lbcs.local @@ -0,0 +1,6 @@ +#%Module + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/make_grid.local b/modulefiles/tasks/noaacloud/make_grid.local new file mode 100644 index 0000000000..a97b9e4e58 --- /dev/null +++ b/modulefiles/tasks/noaacloud/make_grid.local @@ -0,0 +1,12 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/make_ics.local b/modulefiles/tasks/noaacloud/make_ics.local new file mode 100644 index 0000000000..ea7d89b73a --- /dev/null +++ b/modulefiles/tasks/noaacloud/make_ics.local @@ -0,0 +1,14 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran +module load libpng +module load jasper + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/make_lbcs.local b/modulefiles/tasks/noaacloud/make_lbcs.local new file mode 100644 index 0000000000..ea7d89b73a --- /dev/null +++ b/modulefiles/tasks/noaacloud/make_lbcs.local @@ -0,0 +1,14 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran +module load libpng +module load jasper + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/make_orog.local b/modulefiles/tasks/noaacloud/make_orog.local new file mode 100644 index 0000000000..a97b9e4e58 --- /dev/null +++ b/modulefiles/tasks/noaacloud/make_orog.local @@ -0,0 +1,12 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/make_sfc_climo.local b/modulefiles/tasks/noaacloud/make_sfc_climo.local new file mode 100644 index 0000000000..a97b9e4e58 --- /dev/null +++ b/modulefiles/tasks/noaacloud/make_sfc_climo.local @@ -0,0 +1,12 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/miniconda_regional_workflow b/modulefiles/tasks/noaacloud/miniconda_regional_workflow new file mode 100644 index 0000000000..936a9d4c84 --- /dev/null +++ b/modulefiles/tasks/noaacloud/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /contrib/GST/miniconda3/modulefiles +module load miniconda3/4.10.3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/run_fcst.local b/modulefiles/tasks/noaacloud/run_fcst.local new file mode 100644 index 0000000000..ea7d89b73a --- /dev/null +++ b/modulefiles/tasks/noaacloud/run_fcst.local @@ -0,0 +1,14 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran +module load libpng +module load jasper + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/noaacloud/run_post.local b/modulefiles/tasks/noaacloud/run_post.local new file mode 100644 index 0000000000..ea7d89b73a --- /dev/null +++ b/modulefiles/tasks/noaacloud/run_post.local @@ -0,0 +1,14 @@ +#%Module + +module use /contrib/spack-stack/apps/srw-app-test/modulefiles/Core +module load stack-intel +module load stack-intel-oneapi-mpi +module load netcdf-c +module load netcdf-fortran +module load libpng +module load jasper + +module load miniconda_regional_workflow +module load rocoto +prepend-path PATH /contrib/GST/miniconda/envs/regional_workflow/bin +setenv SRW_ENV regional_workflow diff --git a/ush/machine/noaacloud.sh b/ush/machine/noaacloud.sh index 49299c325f..62a698da8a 100755 --- a/ush/machine/noaacloud.sh +++ b/ush/machine/noaacloud.sh @@ -1,5 +1,6 @@ #!/bin/bash +set -x function file_location() { @@ -19,7 +20,7 @@ function file_location() { esac echo ${location:-} } - +export PROJ_LIB=/contrib/GST/miniconda/envs/regional_workflow/share/proj export OPT=/contrib/EPIC/hpc-modules export PATH=${PATH}:/contrib/GST/miniconda/envs/regional_workflow/bin @@ -65,6 +66,9 @@ RUN_CMD_UTILS='mpiexec -np $nprocs' RUN_CMD_FCST='mpiexec -np ${PE_MEMBER01}' RUN_CMD_POST='mpiexec -np $nprocs' +export build_mod_fn="wflow_noaacloud" +BUILD_MOD_FN="wflow_noaacloud" + # MET Installation Locations # MET Plus is not yet supported on noaacloud . /contrib/EPIC/.bash_conda From 1d3b45feb16391dddcb91ce67037136d5141cc8b Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Thu, 19 May 2022 14:34:03 -0400 Subject: [PATCH 174/203] removed cheyenne-specific crontab editing section (#773) --- ush/get_crontab_contents.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ush/get_crontab_contents.sh b/ush/get_crontab_contents.sh index 9aae441156..182cf19365 100644 --- a/ush/get_crontab_contents.sh +++ b/ush/get_crontab_contents.sh @@ -64,15 +64,6 @@ function get_crontab_contents() { __crontab_contents__=$( ${__crontab_cmd__} -l ) fi # - # On Cheyenne, the output of the "crontab -l" command contains a 3-line - # header (comments) at the top that is not actually part of the user's - # cron table. This needs to be removed to avoid adding an unnecessary - # copy of this header to the user's cron table. - # - if [ "$MACHINE" = "CHEYENNE" ]; then - __crontab_contents__=$( printf "%s" "${__crontab_contents__}" | tail -n +4 ) - fi - # # Set output variables. # printf -v ${outvarname_crontab_cmd} "%s" "${__crontab_cmd__}" From 829f064b2e49cf749c44f10d90e7ef5a5ecfc40d Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Thu, 19 May 2022 15:10:47 -0600 Subject: [PATCH 175/203] Pin down hera miniconda3 module file version. (#770) Pin down the version of miniconda3 on Hera, and do not append to the module path. --- modulefiles/tasks/hera/get_obs.local | 4 ++-- modulefiles/tasks/hera/miniconda_regional_workflow | 4 ++-- tests/WE2E/setup_WE2E_tests.sh | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/modulefiles/tasks/hera/get_obs.local b/modulefiles/tasks/hera/get_obs.local index c41c17de83..d30a2a9189 100644 --- a/modulefiles/tasks/hera/get_obs.local +++ b/modulefiles/tasks/hera/get_obs.local @@ -7,7 +7,7 @@ module purge module load hpss -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module use /contrib/miniconda3/modulefiles +module load miniconda3/4.5.12 setenv SRW_ENV pygraf diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow b/modulefiles/tasks/hera/miniconda_regional_workflow index 011a832c9b..48de7a99bb 100644 --- a/modulefiles/tasks/hera/miniconda_regional_workflow +++ b/modulefiles/tasks/hera/miniconda_regional_workflow @@ -1,5 +1,5 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module use /contrib/miniconda3/modulefiles +module load miniconda3/4.5.12 setenv SRW_ENV regional_workflow diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 1aad100549..de6cdf0c24 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -76,9 +76,12 @@ exec_subdir='bin_intel/bin' #----------------------------------------------------------------------- # Load Python Modules -env_file="${SRW_APP_DIR}/env/wflow_${machine}.env" -source ${env_file} +env_path="${SRW_APP_DIR}/modulefiles" +env_file="wflow_${machine}" echo "-- Load environment =>" $env_file +module use ${env_path} +module load ${env_file} +conda activate regional_workflow module list From 68fa6793b13d5eb4522077d1d06913413d07e1e8 Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Fri, 20 May 2022 12:44:32 -0500 Subject: [PATCH 176/203] update staged data dir (#774) --- ush/machine/gaea.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 37f0838e6f..88f7153665 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -48,7 +48,7 @@ QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} WTIME_MAKE_LBCS="00:60:00" # UFS SRW App specific paths -staged_data_dir="/lustre/f2/dev/Mark.Potts/EPIC/UFS_SRW_App/develop" +staged_data_dir="/lustre/f2/pdata/ncep/UFS_SRW_App/develop" FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} From 681a37ac6c074b11a0ebc2e8bc125c00cd69705d Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Thu, 26 May 2022 12:45:38 -0500 Subject: [PATCH 177/203] update IC/LBC file paths to match new IC/LBC file structure (#766) * update IC/LBC file paths to match new IC/LBC file structure * change forecast length * Update config.community.sh These changes reflects the SRW team's preference for a 12-hours forecast. The gfs.t18z.pgrb2.0p25.f012 file will need to be added to all Level 1 systems. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- ush/config.community.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ush/config.community.sh b/ush/config.community.sh index ba8aa44da2..e483a4b597 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -15,12 +15,12 @@ DO_ENSEMBLE="FALSE" NUM_ENS_MEMBERS="2" CCPP_PHYS_SUITE="FV3_GFS_v16" -FCST_LEN_HRS="48" +FCST_LEN_HRS="12" LBC_SPEC_INTVL_HRS="6" DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" -CYCL_HRS=( "00" ) +CYCL_HRS=( "18" ) EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" @@ -28,7 +28,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" FV3GFS_FILE_FMT_LBCS="grib2" -WTIME_RUN_FCST="01:00:00" +WTIME_RUN_FCST="02:00:00" MODEL="FV3_GFS_v16_CONUS_25km" METPLUS_PATH="path/to/METPlus" @@ -59,8 +59,7 @@ RUN_TASK_VX_ENSPOINT="FALSE" # if on another platform, using other dates, other external models, etc. # Uncomment the following EXTRN_MDL_*_ICS/LBCS only when USE_USER_STAGED_EXTRN_FILES=TRUE # -#EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" -#EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) -#EXTRN_MDL_SOURCE_BASEDIR_LBCS="/path/to/model_data/FV3GFS" -#EXTRN_MDL_FILES_LBCS=( "gfs.pgrb2.0p25.f006" "gfs.pgrb2.0p25.f012" "gfs.pgrb2.0p25.f018" "gfs.pgrb2.0p25.f024" \ -# "gfs.pgrb2.0p25.f030" "gfs.pgrb2.0p25.f036" "gfs.pgrb2.0p25.f042" "gfs.pgrb2.0p25.f048" ) +#EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_App/develop/input_model_data/FV3GFS/grib2/2019061518" +#EXTRN_MDL_FILES_ICS=( "gfs.t18z.pgrb2.0p25.f000" ) +#EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_App/develop/input_model_data/FV3GFS/grib2/2019061518" +#EXTRN_MDL_FILES_LBCS=( "gfs.t18z.pgrb2.0p25.f006" "gfs.t18z.pgrb2.0p25.f012") From afe98ddc7530f85d3b34f2a51f02d9be6cc91863 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 26 May 2022 13:54:37 -0400 Subject: [PATCH 178/203] add a machine file for MacOS (#777) Co-authored-by: Natalie Perlin --- ush/machine/macos.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 ush/machine/macos.sh diff --git a/ush/machine/macos.sh b/ush/machine/macos.sh new file mode 100644 index 0000000000..42d70c5684 --- /dev/null +++ b/ush/machine/macos.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/Users/username/DATA/UFS/FV3GFS/' + ;; + *) + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on MacOSX. " + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + + System scripts to source to initialize various commands within workflow + scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="none" +NCORES_PER_NODE=${NCORES_PER_NODE:-8} +SCHED=${SCHED:-"none"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/Users/username/DATA/UFS/fix/fix_am"} +FIXaer=${FIXaer:-"/Users/username/DATA/UFS/fix/fix_aer"} +FIXlut=${FIXlut:-"/Users/username/DATA/UFS/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/Users/username/DATA/UFS/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/Users/username/DATA/UFS/fix/fix_sfc_climo"} + +# Run commands for executables +RUN_CMD_SERIAL="time" +#Run Commands currently differ for GNU/openmpi +RUN_CMD_UTILS='mpirun -n 4' +RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} ' +RUN_CMD_POST='mpirun -n 4 ' + +# MET Installation Locations + From d0b3a98a4f041e00e6782300a16255c3cfbc90ca Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Tue, 31 May 2022 13:46:29 -0600 Subject: [PATCH 179/203] Add columns for relative test and time step to CSV file containing WE2E test info (#776) ## DESCRIPTION OF CHANGES: This PR modifies the script `get_WE2Etest_names_subdirs_descs.sh` that creates the CSV file containing test information the columns for relative cost and time step (`DT_ATMOS`). This allows users to have a better idea of what the cost of each test is (the relative cost is defined such that 1 unit represents the cost of running a single 6-hour forecast on the `RRFS_CONUS_25km` grid). Other related changes: * Add the flag `generate_csv_file` to the `run_WE2E_tests.sh` script to allow users to skip the CSV file generation step (since it can take a while). * Use arguments for the function `set_predef_grid_params()` so that it can be called not just from the experiment generation scripts but also others (in this case from `get_WE2Etest_names_subdirs_descs.sh`). Necessary for this PR. * Move some constants (`NH0`, `NH3`, `NH4`) from `config_defaults.sh` to `constants.sh` to make it easier to access them. * Place the contents of `constants.sh` at the start of each experiment's `var_defns.sh` so that these constants are available to the j-jobs and ex-scripts when `var_defns.sh` is sourced. * Use variable `valid_vals_BOOLEAN` in many more places (which allows many other `valid_vals_...` variables in `valid_param_vals.sh` and `setup.sh` to be removed). * Several other bug fixes. ## TESTS CONDUCTED: The following tests (with results) were conducted on Hera. There were only three (previously existing) failures. ``` * grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16: SUCCESS * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp: SUCCESS * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional: SUCCESS * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2: SUCCESS * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16: SUCCESS * grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR: SUCCESS * grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp: SUCCESS * grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2: SUCCESS * grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16: FAILURE * grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR: SUCCESS * grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta: SUCCESS * grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16: SUCCESS * grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR: SUCCESS * grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta: SUCCESS * grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR: SUCCESS * grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha: SUCCESS * grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta: SUCCESS * grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16: SUCCESS * grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_HRRR: SUCCESS * grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta: SUCCESS * GST_release_public_v1: SUCCESS * MET_ensemble_verification: SUCCESS * MET_verification: SUCCESS * community_ensemble_008mems: SUCCESS * community_ensemble_2mems: SUCCESS * custom_ESGgrid: SUCCESS * custom_GFDLgrid: SUCCESS * custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE: SUCCESS * custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE: SUCCESS * deactivate_tasks: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100: SUCCESS * get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000: SUCCESS * get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS: SUCCESS * get_from_HPSS_ics_HRRR_lbcs_RAP: SUCCESS * get_from_HPSS_ics_RAP_lbcs_RAP: SUCCESS * get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio: SUCCESS * inline_post: SUCCESS * nco_ensemble: SUCCESS * nco_inline_post: SUCCESS * pregen_grid_orog_sfc_climo: SUCCESS * specify_DOT_OR_USCORE: SUCCESS * specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE: SUCCESS * specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS: SUCCESS * specify_RESTART_INTERVAL: SUCCESS * specify_template_filenames: SUCCESS * subhourly_post: FAILURE * subhourly_post_ensemble_2mems: FAILURE ``` ## DOCUMENTATION: Documentation is updated via PR #[278](https://github.com/ufs-community/ufs-srweather-app/pull/278) in the ``ufs-srweather-app`` repo. --- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 550 ++++- tests/WE2E/get_expts_status.sh | 3 +- tests/WE2E/run_WE2E_tests.sh | 163 +- .../wflow_features/config.custom_ESGgrid.sh | 2 + ush/bash_utils/get_bash_file_contents.sh | 4 +- ush/check_expt_config_vars.sh | 4 +- ush/config_defaults.sh | 16 +- ush/constants.sh | 46 +- ush/get_crontab_contents.sh | 2 +- ush/launch_FV3LAM_wflow.sh | 4 +- ush/load_modules_run_task.sh | 4 +- ush/set_gridparams_ESGgrid.sh | 119 +- ush/set_gridparams_GFDLgrid.sh | 119 +- ush/set_predef_grid_params.sh | 1897 ++++++++++------- ush/setup.sh | 372 ++-- ush/valid_param_vals.sh | 33 +- 16 files changed, 2090 insertions(+), 1248 deletions(-) diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index d0db858b33..57262e9261 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -27,29 +27,29 @@ # Optional verbosity flag. Should be set to "TRUE" or "FALSE". Default # is "FALSE". # -# output_varname_test_configs_basedir: +# outvarname_test_configs_basedir: # Name of output variable in which to return the base directory of the # WE2E test configuration files. # -# output_varname_test_names: +# outvarname_test_names: # Name of output array variable in which to return the names of the WE2E # tests. # -# output_varname_test_subdirs: +# outvarname_test_subdirs: # Name of output array variable in which to return the category subdirectories # in which the WE2E tests are located. # -# output_varname_test_ids: +# outvarname_test_ids: # Name of output array variable in which to return the IDs of the WE2E # tests. # -# output_varname_test_descs: +# outvarname_test_descs: # Name of output array variable in which to return the descriptions of # the WE2E tests. # # Note that any input argument that is not specified in the call to this # function gets set to a null string in the body of the function. In -# particular, if any of the arguments that start with "output_varname_" +# particular, if any of the arguments that start with "outvarname_" # (indicating that they specify the name of an output variable) are not # set in the call, the values corresponding to those variables are not # returned to the calling script or function. @@ -61,10 +61,10 @@ # # test_configs_basedir="${WE2Edir}/test_configs" # -# If the argument output_varname_test_configs_basedir is specified in -# the call to this function, then the value of test_configs_basedir will -# be returned to the calling script or function (in the variable specified -# by output_varname_test_configs_basedir). +# If the argument outvarname_test_configs_basedir is specified in the +# call to this function, then the value of test_configs_basedir will be +# returned to the calling script or function (in the variable specified +# by outvarname_test_configs_basedir). # # The WE2E test configuration files are located in subdirectories under # the base directory. This function sets the names of these subdirectories @@ -170,8 +170,8 @@ # in which the test configuration files corresponding to each test # name are located. # 3) The IDs corresponding to each of the test names. -# 4) The test descriptions (if output_varname_test_descs is specified in -# the call to this function or if generate_csv_file is or gets set to +# 4) The test descriptions (if outvarname_test_descs is specified in the +# call to this function or if generate_csv_file is or gets set to # "TRUE"; see below). # # These local arrays are sorted in order of increasing test ID. Within @@ -189,10 +189,10 @@ # the arrays in which each of the quantities listed above should be # returned (to the calling script or function): # -# output_varname_test_names -# output_varname_test_subdirs -# output_varname_test_ids -# output_varname_test_descs +# outvarname_test_names +# outvarname_test_subdirs +# outvarname_test_ids +# outvarname_test_descs # # If any of these is not specified in the call to this function, then # the corresponding quantity will not be returned to the calling script @@ -206,7 +206,7 @@ # 1) The user explicitly asks for the descriptions to be returned by # specifying in the call to this function the name of the array in # which to return them (by setting a value for the argument -# output_varname_test_descs). +# outvarname_test_descs). # 2) A CSV file summarizing the WE2E tests will be generated (see below) # # For convenience, this function can generate a CSV (comma-separated @@ -267,11 +267,11 @@ function get_WE2Etest_names_subdirs_descs() { "WE2Edir" \ "generate_csv_file" \ "verbose" \ - "output_varname_test_configs_basedir" \ - "output_varname_test_names" \ - "output_varname_test_subdirs" \ - "output_varname_test_ids" \ - "output_varname_test_descs" \ + "outvarname_test_configs_basedir" \ + "outvarname_test_names" \ + "outvarname_test_subdirs" \ + "outvarname_test_ids" \ + "outvarname_test_descs" \ ) process_args "valid_args" "$@" # @@ -294,7 +294,7 @@ function get_WE2Etest_names_subdirs_descs() { # verbose=${verbose:-"FALSE"} check_var_valid_value "verbose" "valid_vals_BOOLEAN" - verbose=$(boolify $verbose) + verbose=$(boolify "$verbose") # #----------------------------------------------------------------------- # @@ -302,7 +302,9 @@ function get_WE2Etest_names_subdirs_descs() { # #----------------------------------------------------------------------- # - local all_items \ + local abs_cost_ref \ + ac \ + all_items \ alt_test_name \ alt_test_names \ alt_test_names_subdirs \ @@ -316,11 +318,15 @@ function get_WE2Etest_names_subdirs_descs() { column_titles \ config_fn \ crnt_item \ + crnt_title \ csv_delimiter \ csv_fn \ csv_fp \ cwd \ default_val \ + dt_atmos \ + fcst_len_hrs \ + get_test_descs \ hash_or_null \ i \ ii \ @@ -331,6 +337,7 @@ function get_WE2Etest_names_subdirs_descs() { mod_time_csv \ mod_time_subdir \ msg \ + nf \ num_alt_tests \ num_category_subdirs \ num_cdates \ @@ -338,20 +345,25 @@ function get_WE2Etest_names_subdirs_descs() { num_days \ num_fcsts \ num_fcsts_orig \ + num_grid_pts \ num_items \ num_occurrences \ num_prim_tests \ num_tests \ + num_time_steps \ num_vars_to_extract \ prim_array_names_vars_to_extract \ prim_test_descs \ + prim_test_dt_atmos \ prim_test_ids \ prim_test_name_subdir \ prim_test_names \ prim_test_num_fcsts \ + prim_test_rel_cost \ prim_test_subdirs \ - get_test_descs \ + rc \ regex_search \ + rel_cost \ row_content \ sort_inds \ stripped_line \ @@ -386,11 +398,49 @@ function get_WE2Etest_names_subdirs_descs() { test_subdirs_orig \ test_subdirs_str \ test_type \ + units \ + ushdir \ val \ - valid_vals_generate_csv_file \ var_name \ var_name_at \ vars_to_extract + + local grid_gen_method \ + \ + gfdlgrid_lon_t6_ctr \ + gfdlgrid_lat_t6_ctr \ + gfdlgrid_res \ + gfdlgrid_stretch_fac \ + gfdlgrid_refine_ratio \ + gfdlgrid_istart_of_rgnl_dom_on_t6g \ + gfdlgrid_iend_of_rgnl_dom_on_t6g \ + gfdlgrid_jstart_of_rgnl_dom_on_t6g \ + gfdlgrid_jend_of_rgnl_dom_on_t6g \ + \ + esggrid_lon_ctr \ + esggrid_lat_ctr \ + esggrid_nx \ + esggrid_ny \ + esggrid_pazi \ + esggrid_wide_halo_width \ + esggrid_delx \ + esggrid_dely \ + \ + nx \ + ny \ + dta +# +#----------------------------------------------------------------------- +# +# Source files. +# +#----------------------------------------------------------------------- +# + ushdir=$( readlink -f "$WE2Edir/../../ush" ) + . $ushdir/constants.sh + . $ushdir/set_predef_grid_params.sh + . $ushdir/set_gridparams_GFDLgrid.sh + . $ushdir/set_gridparams_ESGgrid.sh # #----------------------------------------------------------------------- # @@ -410,17 +460,8 @@ function get_WE2Etest_names_subdirs_descs() { # if [ ! -z "${generate_csv_file}" ]; then - valid_vals_generate_csv_file=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") - check_var_valid_value "generate_csv_file" "valid_vals_generate_csv_file" - - generate_csv_file=${generate_csv_file^^} - if [ "${generate_csv_file}" = "TRUE" ] || \ - [ "${generate_csv_file}" = "YES" ]; then - generate_csv_file="TRUE" - elif [ "${generate_csv_file}" = "FALSE" ] || \ - [ "${generate_csv_file}" = "NO" ]; then - generate_csv_file="FALSE" - fi + check_var_valid_value "generate_csv_file" "valid_vals_BOOLEAN" + generate_csv_file=$(boolify "${generate_csv_file}") # # If generate_csv_file was not specified as an input argument in the # call to this function, then it will have been set above to a null @@ -521,6 +562,8 @@ information on all WE2E tests: prim_test_ids=() prim_test_subdirs=() prim_test_num_fcsts=() + prim_test_dt_atmos=() + prim_test_rel_cost=() alt_test_names=() alt_test_subdirs=() @@ -859,18 +902,18 @@ they correspond to unique test names and rerun." # #----------------------------------------------------------------------- # -# If the input argument output_varname_test_descs is not set to a null -# string (meaning that the name of the array in which to return the WE2E -# test descriptions is specified in the call to this function), or if -# the flag generate_csv_file is set to "TRUE", we need to obtain the -# WE2E test descriptions from the test configuration files. In these -# cases, set the local variable get_test_descs to "TRUE". Otherwise, -# set it to "FALSE". +# If the input argument outvarname_test_descs is not set to a null string +# (meaning that the name of the array in which to return the WE2E test +# descriptions is specified in the call to this function), or if the flag +# generate_csv_file is set to "TRUE", we need to obtain the WE2E test +# descriptions from the test configuration files. In these cases, set +# the local variable get_test_descs to "TRUE". Otherwise, set it to +# "FALSE". # #----------------------------------------------------------------------- # get_test_descs="FALSE" - if [ ! -z "${output_varname_test_descs}" ] || \ + if [ ! -z "${outvarname_test_descs}" ] || \ [ "${generate_csv_file}" = "TRUE" ]; then get_test_descs="TRUE" fi @@ -924,9 +967,9 @@ they correspond to unique test names and rerun." prim_array_names_vars_to_extract=( $( printf "prim_test_%s_vals " "${vars_to_extract[@]}" ) ) array_names_vars_to_extract=( $( printf "%s_vals " "${vars_to_extract[@]}" ) ) for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do - cmd="${prim_array_names_vars_to_extract[$k]}=()" + cmd="local ${prim_array_names_vars_to_extract[$k]}=()" eval $cmd - cmd="${array_names_vars_to_extract[$k]}=()" + cmd="local ${array_names_vars_to_extract[$k]}=()" eval $cmd done @@ -1117,20 +1160,221 @@ ${test_desc}${stripped_line} " # # Calculate the number of forecasts that will be launched by the current # test. The "10#" forces bash to treat the following number as a decimal -# (not hexadecimal, etc). +# (not hexadecimal, etc). Note that INCR_CYCL_FREQ is in units of hours, +# so the factor of 24 is needed to convert the number of days to hours. # num_cycles_per_day=${#CYCL_HRS[@]} num_days=$(( (${DATE_LAST_CYCL} - ${DATE_FIRST_CYCL} + 1)*24/10#${INCR_CYCL_FREQ} )) num_cdates=$(( ${num_cycles_per_day}*${num_days} )) nf=$(( ${num_cdates}*10#${NUM_ENS_MEMBERS} )) # -# In the following, the single quote at the beginning forces Google Sheets -# to interpret this quantity as a string. This prevents any automatic -# number fomatting from being applied when the CSV file is imported into -# Google Sheets. +# Save the number of forecasts launched by the current test in an +# appropriately named array. In the following, the single quote at the +# beginning forces Google Sheets to interpret this quantity as a string. +# This prevents any automatic number fomatting from being applied when +# the CSV file is imported into Google Sheets. # prim_test_num_fcsts+=( "'$nf" ) # +#----------------------------------------------------------------------- +# +# Calculate the relative dynamics cost of the test, i.e. the relative +# cost of running only the dynamics portion of the forecast model. Here, +# we define the absolute cost of running the dynamics as +# +# abs_cost = nx*ny*num_time_steps*num_fcsts +# +# where nx and ny are the horizontal dimensions of the grid, num_time_steps +# is the number of time steps that need to be taken to complete one +# forecast within the test, and num_fcsts are the number of forecasts +# the test makes (e.g. if the test performs an ensemble forecast, the +# value of this parameter will be greater than 1). +# +# The relative cost is obtained by dividing the absolute cost of a test +# by the absolute cost of a reference 6-hour forecast on the RRFS_CONUS_25km +# predefined grid using the default time step for that grid. This is +# calculated later below and saved in the variable abs_cost_ref. Thus, +# the relative cost is given by +# +# rel_cost = abs_cost/abs_cost_ref +# +# defined as abs_cost_ref. +# +# Note that the (absolute or relative) cost defined here does not take +# into account the costs of running different physics suites, nor does +# it take into account the costs of workflow tasks other than the forecast +# task (e.g. generation of initial and boundary conditions, post processing, +# verification, etc; that is why it is referred to as the relative DYNAMICS +# cost). Note also that if in the future the number of levels in the +# vertical becomes a user-specified parameter, that will also have to be +# added to the definition of the cost. +# +#----------------------------------------------------------------------- +# + +# +# To calculate the absolute cost as defined above, we need the number of +# points in the two horizontal directions, nx and ny. Also, to calculate +# the number of time steps, we need the size of the time step (dt_atmos). +# These depend on the grid being used and must be extracted from the grid +# parameters. The way the latter are obtained depends on whether or not +# a predefined grid is being used. +# +# If using a predefined grid, call the set_predef_grid_params() function +# to get the grid parameters. +# + if [ ! -z "${PREDEF_GRID_NAME}" ]; then +# +# Note: +# Can set "quilting" to "FALSE" in the following argument list because +# the write-component parameters are not needed below; only those of the +# native grid are needed. +# + set_predef_grid_params \ + predef_grid_name="${PREDEF_GRID_NAME}" \ + quilting="FALSE" \ + outvarname_grid_gen_method="grid_gen_method" \ + outvarname_esggrid_lon_ctr="esggrid_lon_ctr" \ + outvarname_esggrid_lat_ctr="esggrid_lat_ctr" \ + outvarname_esggrid_delx="esggrid_delx" \ + outvarname_esggrid_dely="esggrid_dely" \ + outvarname_esggrid_nx="esggrid_nx" \ + outvarname_esggrid_ny="esggrid_ny" \ + outvarname_esggrid_pazi="esggrid_pazi" \ + outvarname_esggrid_wide_halo_width="esggrid_wide_halo_width" \ + outvarname_gfdlgrid_lon_t6_ctr="gfdlgrid_lon_t6_ctr" \ + outvarname_gfdlgrid_lat_t6_ctr="gfdlgrid_lat_t6_ctr" \ + outvarname_gfdlgrid_stretch_fac="gfdlgrid_stretch_fac" \ + outvarname_gfdlgrid_res="gfdlgrid_res" \ + outvarname_gfdlgrid_refine_ratio="gfdlgrid_refine_ratio" \ + outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="gfdlgrid_istart_of_rgnl_dom_on_t6g" \ + outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="gfdlgrid_iend_of_rgnl_dom_on_t6g" \ + outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g="gfdlgrid_jstart_of_rgnl_dom_on_t6g" \ + outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g="gfdlgrid_jend_of_rgnl_dom_on_t6g" \ + outvarname_dt_atmos="dta" +# +# If using a custom grid, the test's configuration file should contain +# the grid parameters. Source this file and set the values of the grid +# parameters it contains to local variables. +# + else + + . ./${config_fn} + grid_gen_method="${GRID_GEN_METHOD}" + if [ "${grid_gen_method}" = "GFDLgrid" ]; then + gfdlgrid_lon_t6_ctr="${GFDLgrid_LON_T6_CTR}" + gfdlgrid_lat_t6_ctr="${GFDLgrid_LAT_T6_CTR}" + gfdlgrid_res="${GFDLgrid_RES}" + gfdlgrid_stretch_fac="${GFDLgrid_STRETCH_FAC}" + gfdlgrid_refine_ratio="${GFDLgrid_REFINE_RATIO}" + gfdlgrid_istart_of_rgnl_dom_on_t6g="${GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G}" + gfdlgrid_iend_of_rgnl_dom_on_t6g="${GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G}" + gfdlgrid_jstart_of_rgnl_dom_on_t6g="${GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G}" + gfdlgrid_jend_of_rgnl_dom_on_t6g="${GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G}" + elif [ "${grid_gen_method}" = "ESGgrid" ]; then + esggrid_lon_ctr="${ESGgrid_LON_CTR}" + esggrid_lat_ctr="${ESGgrid_LAT_CTR}" + esggrid_delx="${ESGgrid_DELX}" + esggrid_dely="${ESGgrid_DELY}" + esggrid_nx="${ESGgrid_NX}" + esggrid_ny="${ESGgrid_NY}" + esggrid_pazi="${ESGgrid_PAZI}" + esggrid_wide_halo_width="${ESGgrid_WIDE_HALO_WIDTH}" + fi + dta="${DT_ATMOS}" + + fi +# +# Save the value of dta (which is just dt_atmos) in an array. The single +# quote at the beginning forces Google Sheets to interpret this quantity +# as a string. This prevents any automatic number fomatting from being +# applied when the CSV file is imported into Google Sheets. +# + prim_test_dt_atmos+=( "'${dta}" ) +# +# The way the number of grid points in the horizontal directions (nx and +# ny) are calculated depends on the method used to generate the grid as +# well as the grid parameters for that method. +# + if [ "${grid_gen_method}" = "GFDLgrid" ]; then +# +# Note: +# The workflow generation mode (run_envir) can be set to "community" here +# since this does not affect the values of nx and ny. +# + set_gridparams_GFDLgrid \ + lon_of_t6_ctr="${gfdlgrid_lon_t6_ctr}" \ + lat_of_t6_ctr="${gfdlgrid_lat_t6_ctr}" \ + res_of_t6g="${gfdlgrid_res}" \ + stretch_factor="${gfdlgrid_stretch_fac}" \ + refine_ratio_t6g_to_t7g="${gfdlgrid_refine_ratio}" \ + istart_of_t7_on_t6g="${gfdlgrid_istart_of_rgnl_dom_on_t6g}" \ + iend_of_t7_on_t6g="${gfdlgrid_iend_of_rgnl_dom_on_t6g}" \ + jstart_of_t7_on_t6g="${gfdlgrid_jstart_of_rgnl_dom_on_t6g}" \ + jend_of_t7_on_t6g="${gfdlgrid_jend_of_rgnl_dom_on_t6g}" \ + verbose="$verbose" \ + outvarname_nx_of_t7_on_t7g="nx" \ + outvarname_ny_of_t7_on_t7g="ny" + + elif [ "${grid_gen_method}" = "ESGgrid" ]; then + + set_gridparams_ESGgrid \ + lon_ctr="${esggrid_lon_ctr}" \ + lat_ctr="${esggrid_lat_ctr}" \ + nx="${esggrid_nx}" \ + ny="${esggrid_ny}" \ + pazi="${esggrid_pazi}" \ + halo_width="${esggrid_wide_halo_width}" \ + delx="${esggrid_delx}" \ + dely="${esggrid_dely}" \ + outvarname_nx="nx" \ + outvarname_ny="ny" + + fi +# +# Calculate the total number of horizontal grid points. +# + num_grid_pts=$(( nx*ny )) +# +# Calculate the number of time steps for the test. Note that FCST_LEN_HRS +# is in units of hours while dta is in units of seconds. +# + num_time_steps=$(( FCST_LEN_HRS*3600/dta + 1 )) +# +# Calculate the absolute cost of the test. +# + ac=$(( num_grid_pts*num_time_steps*nf )) +# +# Unset all grid paramters so that they are not accidentally reused for +# the next test. +# + unset gfdlgrid_lon_t6_ctr \ + gfdlgrid_lat_t6_ctr \ + gfdlgrid_res \ + gfdlgrid_stretch_fac \ + gfdlgrid_refine_ratio \ + gfdlgrid_istart_of_rgnl_dom_on_t6g \ + gfdlgrid_iend_of_rgnl_dom_on_t6g \ + gfdlgrid_jstart_of_rgnl_dom_on_t6g \ + gfdlgrid_jend_of_rgnl_dom_on_t6g \ + esggrid_lon_ctr \ + esggrid_lat_ctr \ + esggrid_nx \ + esggrid_ny \ + esggrid_pazi \ + esggrid_wide_halo_width \ + esggrid_delx \ + esggrid_dely \ + dta \ + nx \ + ny +# +# Save the absolute cost for this test in the array that will eventually +# contain the relative cost. The values in this array will be divided +# by abs_cost_ref later below to obtain relative costs. +# + prim_test_rel_cost+=( "$ac" ) +# # Unset the experiment variables defined for the current test so that # they are not accidentally used for the next one. # @@ -1140,6 +1384,37 @@ ${test_desc}${stripped_line} " eval $cmd done + done # End loop over primary tests +# +#----------------------------------------------------------------------- +# +# Normalize the absolute costs calculated above for each test by the +# absolute cost of a reference 6-hour forecast on the RRFS_CONUS_25km +# predefined grid (using the default time step for that grid). +# +#----------------------------------------------------------------------- +# + set_predef_grid_params \ + predef_grid_name="RRFS_CONUS_25km" \ + quilting="FALSE" \ + outvarname_esggrid_nx="nx" \ + outvarname_esggrid_ny="ny" \ + outvarname_dt_atmos="dta" + + num_grid_pts=$(( nx*ny )) + fcst_len_hrs="6" + num_time_steps=$(( fcst_len_hrs*3600/dta+ 1 )) + abs_cost_ref=$(( num_grid_pts*num_time_steps )) + + for (( i=0; i<=$((num_prim_tests-1)); i++ )); do +# +# In the following, the single quote at the beginning forces Google Sheets +# to interpret this quantity as a string. This prevents any automatic +# number fomatting from being applied when the CSV file is imported into +# Google Sheets. +# + prim_test_rel_cost[$i]="'"$( printf "%g" \ + $( bc -l <<< " ${prim_test_rel_cost[$i]}/${abs_cost_ref}" ) ) done fi @@ -1158,6 +1433,8 @@ ${test_desc}${stripped_line} " if [ "${get_test_descs}" = "TRUE" ]; then test_descs=("${prim_test_descs[@]}") num_fcsts=("${prim_test_num_fcsts[@]}") + dt_atmos=("${prim_test_dt_atmos[@]}") + rel_cost=("${prim_test_rel_cost[@]}") for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do cmd="${array_names_vars_to_extract[$k]}=(\"\${${prim_array_names_vars_to_extract[$k]}[@]}\")" eval $cmd @@ -1187,6 +1464,8 @@ ${test_desc}${stripped_line} " if [ "${get_test_descs}" = "TRUE" ]; then test_descs+=("${prim_test_descs[$j]}") num_fcsts+=("${prim_test_num_fcsts[$j]}") + dt_atmos+=("${prim_test_dt_atmos[$j]}") + rel_cost+=("${prim_test_rel_cost[$j]}") for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do cmd="${array_names_vars_to_extract[$k]}+=(\"\${${prim_array_names_vars_to_extract[$k]}[$j]}\")" eval $cmd @@ -1266,9 +1545,9 @@ Please correct and rerun." sed -n -r -e "s/${regex_search}/\2/p" ) done - test_names_orig=( "${test_names[@]}" ) - test_subdirs_orig=( "${test_subdirs[@]}" ) - test_ids_orig=( "${test_ids[@]}" ) + local test_names_orig=( "${test_names[@]}" ) + local test_subdirs_orig=( "${test_subdirs[@]}" ) + local test_ids_orig=( "${test_ids[@]}" ) for (( i=0; i<=$((num_tests-1)); i++ )); do ii="${sort_inds[$i]}" test_names[$i]="${test_names_orig[$ii]}" @@ -1278,10 +1557,12 @@ Please correct and rerun." if [ "${get_test_descs}" = "TRUE" ]; then - test_descs_orig=( "${test_descs[@]}" ) - num_fcsts_orig=( "${num_fcsts[@]}" ) + local test_descs_orig=( "${test_descs[@]}" ) + local num_fcsts_orig=( "${num_fcsts[@]}" ) + local dt_atmos_orig=( "${dt_atmos[@]}" ) + local rel_cost_orig=( "${rel_cost[@]}" ) for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do - cmd="${array_names_vars_to_extract[$k]}_orig=(\"\${${array_names_vars_to_extract[$k]}[@]}\")" + cmd="local ${array_names_vars_to_extract[$k]}_orig=(\"\${${array_names_vars_to_extract[$k]}[@]}\")" eval $cmd done @@ -1289,6 +1570,8 @@ Please correct and rerun." ii="${sort_inds[$i]}" test_descs[$i]="${test_descs_orig[$ii]}" num_fcsts[$i]="${num_fcsts_orig[$ii]}" + dt_atmos[$i]="${dt_atmos_orig[$ii]}" + rel_cost[$i]="${rel_cost_orig[$ii]}" for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do cmd="${array_names_vars_to_extract[$k]}[$i]=\"\${${array_names_vars_to_extract[$k]}_orig[$ii]}\"" eval $cmd @@ -1325,14 +1608,44 @@ Please correct and rerun." # detail further below. # column_titles="\ -\"Test Name (Subdirectory)\" ${csv_delimiter} \ -\"Alternate Test Names (Subdirectories)\" ${csv_delimiter} \ +\"Test Name +(Subdirectory)\" ${csv_delimiter} \ +\"Alternate Test Names +(Subdirectories)\" ${csv_delimiter} \ \"Test Purpose/Description\" ${csv_delimiter} \ +\"Relative Cost of Running Dynamics +(1 corresponds to running a 6-hour forecast on the RRFS_CONUS_25km predefined grid using the default time step)\" ${csv_delimiter} \ \"Number of Forecast Model Runs\"" for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do - column_titles="\ -${column_titles} ${csv_delimiter} \ -\"${vars_to_extract[$k]}\"" + + crnt_title="${vars_to_extract[$k]}" + # + # Add units for select fields. + # + units="" + case "${vars_to_extract[$k]}" in + "INCR_CYCL_FREQ") + units="[hr]" + ;; + "FCST_LEN_HRS") + units="[hr]" + ;; + "LBC_SPEC_INTVL_HRS") + units="[hr]" + ;; + esac + crnt_title="${crnt_title}${units:+ $units}" + + column_titles="${column_titles} ${csv_delimiter} \"${crnt_title}\"" + # + # Insert a column for DT_ATMOS right after the one for FCST_LEN_HRS. + # + if [ "${vars_to_extract[$k]}" = "FCST_LEN_HRS" ]; then + units="[sec]" + crnt_title="DT_ATMOS${units:+ $units}" + column_titles="${column_titles} ${csv_delimiter} \"${crnt_title}\"" + fi + done printf "%s\n" "${column_titles}" >> "${csv_fp}" # @@ -1349,7 +1662,7 @@ ${column_titles} ${csv_delimiter} \ # Get the primary name of the test and the category subdirectory in which # it is located. # - prim_test_name_subdir="${test_names[$j]} (${test_subdirs[$j]})" + prim_test_name_subdir="${test_names[$j]}"$'\n'"(${test_subdirs[$j]})" # # Get the test ID. # @@ -1365,8 +1678,15 @@ ${column_titles} ${csv_delimiter} \ # test_desc=$( printf "%s" "${test_desc}" | sed -r -e "s/\"/\"\"/g" ) # -# Get the number of forecasts (number of times the forcast model is run, -# due to a unique starting date, an ensemble member, etc). +# Get the time step. +# + dta="${dt_atmos[$j]}" +# +# Get the relative cost. +# + rc="${rel_cost[$j]}" +# +# Get the number of forecasts (number of times the forcast model is run). # nf="${num_fcsts[$j]}" # @@ -1382,15 +1702,15 @@ ${column_titles} ${csv_delimiter} \ while [ "$jp1" -lt "${num_tests}" ]; do test_id_next="${test_ids[$jp1]}" if [ "${test_id_next}" -eq "${test_id}" ]; then - alt_test_names_subdirs="\ -${alt_test_names_subdirs} -${test_names[$jp1]} (${test_subdirs[$jp1]})" + alt_test_names_subdirs="${alt_test_names_subdirs}${test_names[$jp1]}"$'\n'"(${test_subdirs[$jp1]})"$'\n' j="$jp1" jp1=$((j+1)) else break fi done +# Remove trailing newline. + alt_test_names_subdirs="${alt_test_names_subdirs%$'\n'}" # # Write a line to the CSV file representing a single row of the spreadsheet. # This row contains the following columns: @@ -1400,35 +1720,54 @@ ${test_names[$jp1]} (${test_subdirs[$jp1]})" # located in (the latter in parentheses). # # Column 2: -# The alternate test names (if any) followed by their subdirectories -# (in parentheses). Each alternate test name and subdirectory pair is -# followed by a newline, but all lines will appear in a single cell of -# the spreadsheet. +# Any alternate test names followed by their category subdirectories (in +# parentheses). Each alternate test name and subdirectory pair is followed +# by a newline, but all lines will appear in a single cell of the spreadsheet. # # Column 3: # The test description. # # Column 4: -# The number of times the forecast model will be run by the test. This -# has been calculated above using the quantities that go in Columns 5, -# 6, .... +# The relative cost of running the dynamics in the test. See above for +# details. # -# Columns 5...: -# The values of the experiment variables specified in vars_to_extract. +# Column 5: +# The number of times the forecast model will be run by the test. This +# is calculated using quantities such as the number of cycle dates (i.e. +# forecast model start dates) and the number of of ensemble members (which +# is greater than 1 if running ensemble forecasts and 1 otherwise). The +# latter are in turn obtained directly or indirectly from the quantities +# in Columns 6, 7, .... +# +# Columns 6, 7, ...: +# The values of the experiment variables specified in vars_to_extract, +# plus DT_ATMOS (included right after FCST_LEN_HRS). Note that DT_ATMOS +# cannot be included in vars_to_extract because it is usually not in the +# WE2E test configuration file where this script looks for these variables +# (because most of the tests use predefined grids, and for those cases, +# DT_ATMOS is defined in the same file/script where the other grid +# parameters are defined). # row_content="\ \"${prim_test_name_subdir}\" ${csv_delimiter} \ \"${alt_test_names_subdirs}\" ${csv_delimiter} \ \"${test_desc}\" ${csv_delimiter} \ +\"${rc}\" ${csv_delimiter} \ \"${nf}\"" for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do + unset "val" cmd="val=\"\${${array_names_vars_to_extract[$k]}[$j]}\"" eval $cmd - row_content="\ -${row_content} ${csv_delimiter} \ -\"${val}\"" + row_content="${row_content} ${csv_delimiter} \"${val}\"" +# +# Insert value of DT_ATMOS right after value of FCST_LEN_HRS. +# + if [ "${vars_to_extract[$k]}" = "FCST_LEN_HRS" ]; then + row_content="${row_content} ${csv_delimiter} \"${dta}\"" + fi + done printf "%s\n" "${row_content}" >> "${csv_fp}" @@ -1455,41 +1794,40 @@ containing information on all WE2E tests: # #----------------------------------------------------------------------- # - if [ ! -z "${output_varname_test_configs_basedir}" ]; then - eval ${output_varname_test_configs_basedir}="${test_configs_basedir}" + if [ ! -z "${outvarname_test_configs_basedir}" ]; then + eval ${outvarname_test_configs_basedir}="${test_configs_basedir}" fi - if [ ! -z "${output_varname_test_names}" ]; then + if [ ! -z "${outvarname_test_names}" ]; then test_names_str="( "$( printf "\"%s\" " "${test_names[@]}" )")" - eval ${output_varname_test_names}="${test_names_str}" + eval ${outvarname_test_names}="${test_names_str}" fi - if [ ! -z "${output_varname_test_subdirs}" ]; then + if [ ! -z "${outvarname_test_subdirs}" ]; then test_subdirs_str="( "$( printf "\"%s\" " "${test_subdirs[@]}" )")" - eval ${output_varname_test_subdirs}="${test_subdirs_str}" + eval ${outvarname_test_subdirs}="${test_subdirs_str}" fi - if [ ! -z "${output_varname_test_ids}" ]; then + if [ ! -z "${outvarname_test_ids}" ]; then test_ids_str="( "$( printf "\"%s\" " "${test_ids[@]}" )")" - eval ${output_varname_test_ids}="${test_ids_str}" + eval ${outvarname_test_ids}="${test_ids_str}" fi - if [ ! -z "${output_varname_test_descs}" ]; then + if [ ! -z "${outvarname_test_descs}" ]; then # # We want to treat all characters in the test descriptions literally -# when evaluating the array specified by output_varname_test_descs -# below using the eval function because otherwise, characters such as -# "$", "(", ")", etc will be interpreted as indicating the value of a -# variable, the start of an array, the end of an array, etc, and lead to -# errors. Thus, below, when forming the array that will be passed to -# eval, we will surround each element of the local array test_descs -# in single quotes. However, the test descriptions themselves may -# include single quotes (e.g. when a description contains a phrase such -# as "Please see the User's Guide for..."). In order to treat these -# single quotes literally (as opposed to as delimiters indicating the -# start or end of array elements), we have to pass them as separate -# strings by replacing each single quote with the following series of -# characters: +# when evaluating the array specified by outvarname_test_descs below +# using the eval function because otherwise, characters such as "$", +# "(", ")", etc will be interpreted as indicating the value of a variable, +# the start of an array, the end of an array, etc, and lead to errors. +# Thus, below, when forming the array that will be passed to eval, we +# will surround each element of the local array test_descs in single +# quotes. However, the test descriptions themselves may include single +# quotes (e.g. when a description contains a phrase such as "Please see +# the User's Guide for..."). In order to treat these single quotes +# literally (as opposed to as delimiters indicating the start or end of +# array elements), we have to pass them as separate strings by replacing +# each single quote with the following series of characters: # # '"'"' # @@ -1507,8 +1845,8 @@ containing information on all WE2E tests: # # See description of ${DOT_OR_USCORE} in the configuration file. # -# Then, if output_varname_test_descs is set to "some_array", the -# exact string we want to pass to eval is: +# Then, if outvarname_test_descs is set to "some_array", the exact string +# we want to pass to eval is: # # some_array=('Please see the User'"'"'s Guide.' 'See description of ${DOT_OR_USCORE} in the configuration file.') # @@ -1518,7 +1856,7 @@ containing information on all WE2E tests: sed -r -e "s/'/'\"'\"'/g" ) done test_descs_str="( "$( printf "'%s' " "${test_descs_esc_sq[@]}" )")" - eval ${output_varname_test_descs}="${test_descs_str}" + eval ${outvarname_test_descs}="${test_descs_str}" fi # #----------------------------------------------------------------------- diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 91de215d25..a17b4e08fa 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -79,6 +79,7 @@ Usage: ${scrfunc_fn} \\ expts_basedir=\"...\" \\ + [num_log_lines=\"...\"] \\ [verbose=\"...\"] The arguments in brackets are optional. The arguments are defined as @@ -144,7 +145,7 @@ num_log_lines=${num_log_lines:-"40"} # verbose=${verbose:-"FALSE"} check_var_valid_value "verbose" "valid_vals_BOOLEAN" -verbose=$(boolify $verbose) +verbose=$(boolify "$verbose") # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index a11dfaba3a..b2c8c537c7 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -90,6 +90,7 @@ Usage: [use_cron_to_relaunch=\"...\"] \\ [cron_relaunch_intvl_mnts=\"...\"] \\ [verbose=\"...\"] \\ + [generate_csv_file=\"...\"] \\ [machine_file=\"...\"] \\ [stmp=\"...\"] \\ [ptmp=\"...\"] \\ @@ -117,76 +118,84 @@ run. (A description of ACCOUNT can be found in the default experiment configuration file.) This is a required argument. expt_basedir: -Argument used to explicitly set the experiment variable EXPT_BASEDIR in -the experiment configuration files of all the WE2E tests the user wants -to run. (A description of EXPT_BASEDIR can be found in the default -experiment configuration file.) If expt_basedir is specified in the call -to this script, its value is used to set EXPT_BASEDIR in the configuration -files. If it is not specified, EXPT_BASEDIR is not set in the configuration -files, in which case the workflow generation script sets it to a default -value. Note that if expt_basedir is set to a relative path (e.g. -expt_basedir=\"testset1\" in the call to this script), then the workflow -generation script will set EXPT_BASEDIR for the experiment to a default -absolute path followed by \${expt_basedir}. This feature can be used to -group the WE2E tests into subdirectories for convenience, e.g. a set of -tests under subdirectory testset1, another set of tests under testset2, -etc. +Optional argument used to explicitly set the experiment variable +EXPT_BASEDIR in the experiment configuration files of all the WE2E tests +the user wants to run. (A description of EXPT_BASEDIR can be found in +the default experiment configuration file.) If expt_basedir is specified +in the call to this script, its value is used to set EXPT_BASEDIR in the +configuration files. If it is not specified, EXPT_BASEDIR is not set in +the configuration files, in which case the workflow generation script +sets it to a default value. Note that if expt_basedir is set to a +relative path (e.g. expt_basedir=\"testset1\" in the call to this script), +then the experiment generation script will set EXPT_BASEDIR for the +experiment to a default absolute path followed by \${expt_basedir}. +This feature can be used to group the WE2E tests into subdirectories for +convenience, e.g. a set of tests under subdirectory testset1, another +set of tests under testset2, etc. exec_subdir: -Optional. Argument used to set the EXEC_SUBDIR experiment variable. -Please see the default experiment configuration file \"config_defaults.sh\" -for a full description of EXEC_SUBDIR. +Optional argument used to explicitly set the experiment variable +EXEC_SUBDIR in the experiment configuration files of all the WE2E tests +the user wants to run. See the default experiment configuration file +\"config_defaults.sh\" for a full description of EXEC_SUBDIR. use_cron_to_relaunch: -Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH -in the experiment configuration files of all the WE2E tests the user wants -to run. (A description of USE_CRON_TO_RELAUNCH can be found in the default -experiment configuration file.) If use_cron_to_relaunch is specified in -the call to this script, its value is used to set USE_CRON_TO_RELAUNCH -in the configuration files. If it is not specified, USE_CRON_TO_RELAUNCH -is set to \"TRUE\" in the configuration files, in which case cron jobs -are used to (re)launch the workflows for all tests (one cron job per test). -Thus, use_cron_to_relaunch needs to be specified only if the user wants -to turn off use of cron jobs for all tests (by specifying use_cron_to_relaunch= -\"FALSE\" on the command line). Note that it is not possible to specify -a different value for USE_CRON_TO_RELAUNCH for each test via this argument; -either all tests use cron jobs or none do. +Optional argument used to explicitly set the experiment variable +USE_CRON_TO_RELAUNCH in the experiment configuration files of all the +WE2E tests the user wants to run. (A description of USE_CRON_TO_RELAUNCH +can be found in the default experiment configuration file.) If +use_cron_to_relaunch is specified in the call to this script, its value +is used to set USE_CRON_TO_RELAUNCH in the configuration files. If it +is not specified, USE_CRON_TO_RELAUNCH is set to \"TRUE\" in the +configuration files, in which case cron jobs are used to (re)launch the +workflows for all tests (one cron job per test). Thus, use_cron_to_relaunch +needs to be specified only if the user wants to turn off use of cron jobs +for all tests (by specifying use_cron_to_relaunch=\"FALSE\" on the command +line). Note that it is not possible to specify a different value for +USE_CRON_TO_RELAUNCH for each test via this argument; either all tests +use cron jobs or none do. cron_relaunch_intvl_mnts: -Argument used to explicitly set the experiment variable CRON_RELAUNCH_INTVL_MNTS -in the experiment configuration files of all the WE2E tests the user wants -to run. (A description of CRON_RELAUNCH_INTVL_MNTS can be found in the -default experiment configuration file.) If cron_relaunch_intvl_mnts is -specified in the call to this script, its value is used to set -CRON_RELAUNCH_INTVL_MNTS in the configuration files. If it is not -specified, CRON_RELAUNCH_INTVL_MNTS is set to \"02\" (i.e. two minutes) -in the configuration files. Note that it is not possible to specify a -different value for CRON_RELAUNCH_INTVL_MNTS for each test via this -argument; all tests will use the same value for USE_CRON_TO_RELAUNCH -(either the value specified in the call to this script or the default -value of \"02\"). Note also that the value of this argument matters only -if the argument use_cron_to_relaunch is not explicitly set to \"FALSE\" -in the call to this script. +Optional argument used to explicitly set the experiment variable +CRON_RELAUNCH_INTVL_MNTS in the experiment configuration files of +all the WE2E tests the user wants to run. (A description of +CRON_RELAUNCH_INTVL_MNTS can be found in the default experiment +configuration file.) If cron_relaunch_intvl_mnts is specified in the +call to this script, its value is used to set CRON_RELAUNCH_INTVL_MNTS +in the configuration files. If it is not specified, CRON_RELAUNCH_INTVL_MNTS +is set to \"02\" (i.e. two minutes) in the configuration files. Note +that it is not possible to specify a different value for +CRON_RELAUNCH_INTVL_MNTS for each test via this argument; all tests will +use the same value for USE_CRON_TO_RELAUNCH (either the value specified +in the call to this script or the default value of \"02\"). Note also +that the value of this argument matters only if the argument +use_cron_to_relaunch is not explicitly set to \"FALSE\" in the call to +this script. verbose: -Argument used to explicitly set the experiment variable VERBOSE in the -experiment configuration files of all the WE2E tests the user wants to -run. (A description of VERBOSE can be found in the default experiment -configuration file.) If verbose is specified in the call to this script, -its value is used to set VERBOSE in the configuration files. If it is -not specified, VERBOSE is set to \"TRUE\" in the configuration files. -Note that it is not possible to specify a different value for VERBOSE -for each test via this argument; either all tests will have VERBOSE set -to \"TRUE\" or all will have it set to \"FALSE\". +Optional argument used to explicitly set the experiment variable VERBOSE +in the experiment configuration files of all the WE2E tests the user +wants to run. (A description of VERBOSE can be found in the default +experiment configuration file.) If verbose is specified in the call to +this script, its value is used to set VERBOSE in the configuration files. +If it is not specified, VERBOSE is set to \"TRUE\" in the configuration +files. Note that it is not possible to specify a different value for +VERBOSE for each test via this argument; either all tests will have +VERBOSE set to \"TRUE\" or all will have it set to \"FALSE\". + +generate_csv_file: +Optional argument that specifies whether or not to generate a CSV file +containing summary information about all the tests available in the WE2E +testing system. Default value is \"TRUE\". machine_file: -Optional argument to set the full path to a machine configuration file. -If not set, a supported platform machine file may be used. +Optional argument specifying the full path to a machine configuration +file. If not set, a supported platform machine file may be used. stmp: -Argument used to explicitly set the experiment variable STMP in the -experiment configuration files of all the WE2E tests the user wants to -run that are in NCO mode, i.e. they have test configuration files that +Optional argument used to explicitly set the experiment variable STMP in +the experiment configuration files of all the WE2E tests the user wants +to run that are in NCO mode, i.e. they have test configuration files that set the experiment variable RUN_ENVIR to \"nco\". (A description of STMP can be found in the default experiment configuration file.) If stmp is specified in the call to this script, its value is used to set @@ -208,14 +217,23 @@ Same as the argument \"stmp\" described above but for setting the experiment variable PTMP for all tests that will run in NCO mode. compiler: -Type of compiler to use for the workflow. Options are \"intel\" and \"gnu\". -Default is \"intel\". +Optional argument used to explicitly set the experiment variable COMPILER +in the experiment configuration files of all the WE2E tests the user +wants to run. (A description of COMPILER can be found in the default +experiment configuration file.) If compiler is specified in the call to +this script, its value is used to set COMPILER in the configuration files. +If it is not specified, COMPILER is set to \"intel\" in the configuration +files. Note that it is not possible to specify a different value for +COMPILER for each test via this argument; all tests will use the same +value for COMPILER (either the value specified in the call to this script +or the default value of \"intel\"). build_mod_fn: -Specify the build module files (see ufs-srweather-app/modulefiles) to -use for the workflow. (e.g. build_cheyenne_gnu). If a -\"gnu\" compiler is specified, it must also be specified with -the \"compiler\" option. +Optional argument used to explicitly set the experiment variable +BUILD_MOD_FN in the experiment configuration files of all the WE2E tests +the user wants to run (e.g. \"build_cheyenne_gnu\"). If the string +\"gnu\" appears in this file name, the \"compiler\" option to this +function must also be specified with the value \"gnu\". Usage Examples: @@ -320,6 +338,7 @@ valid_args=( \ "use_cron_to_relaunch" \ "cron_relaunch_intvl_mnts" \ "verbose" \ + "generate_csv_file" \ "machine_file" \ "stmp" \ "ptmp" \ @@ -483,7 +502,7 @@ done < "${user_spec_tests_fp}" # avail_WE2E_test_subdirs[7]="dir1" # avail_WE2E_test_subdirs[8]="dir2" # avail_WE2E_test_subdirs[9]="dir3" -# +# #----------------------------------------------------------------------- # print_info_msg " @@ -491,10 +510,11 @@ Getting information about all available WE2E tests..." get_WE2Etest_names_subdirs_descs \ WE2Edir="${WE2Edir}" \ - output_varname_test_configs_basedir="avail_WE2E_test_configs_basedir" \ - output_varname_test_names="avail_WE2E_test_names" \ - output_varname_test_subdirs="avail_WE2E_test_subdirs" \ - output_varname_test_ids="avail_WE2E_test_ids" + generate_csv_file="${generate_csv_file}" \ + outvarname_test_configs_basedir="avail_WE2E_test_configs_basedir" \ + outvarname_test_names="avail_WE2E_test_names" \ + outvarname_test_subdirs="avail_WE2E_test_subdirs" \ + outvarname_test_ids="avail_WE2E_test_ids" # # Get the total number of available WE2E test names (including alternate # names). @@ -759,6 +779,7 @@ Please correct and rerun." BUILD_MOD_FN=${build_mod_fn:-"build_${machine}_${COMPILER}"} EXPT_BASEDIR="${expt_basedir}" EXPT_SUBDIR="${test_name}" + EXEC_SUBDIR="${exec_subdir}" USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} CRON_RELAUNCH_INTVL_MNTS=${cron_relaunch_intvl_mnts:-"02"} VERBOSE=${verbose:-"TRUE"} @@ -783,9 +804,9 @@ ACCOUNT=\"${ACCOUNT}\" COMPILER=\"${COMPILER}\" BUILD_MOD_FN=\"${BUILD_MOD_FN}\"" - if [ -n "${exec_subdir}" ]; then + if [ -n "${EXEC_SUBDIR}" ]; then expt_config_str=${expt_config_str}" -EXEC_SUBDIR=\"${exec_subdir}\"" +EXEC_SUBDIR=\"${EXEC_SUBDIR}\"" fi if [ -n "${EXPT_BASEDIR}" ]; then diff --git a/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh index bb8d2bc0cf..f78fdf7e0d 100644 --- a/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh @@ -36,6 +36,8 @@ ESGgrid_DELY="25000.0" ESGgrid_NX="216" ESGgrid_NY="156" +ESGgrid_PAZI="0.0" + ESGgrid_WIDE_HALO_WIDTH="6" DT_ATMOS="40" diff --git a/ush/bash_utils/get_bash_file_contents.sh b/ush/bash_utils/get_bash_file_contents.sh index 3b3ab7b30d..fdfb626922 100644 --- a/ush/bash_utils/get_bash_file_contents.sh +++ b/ush/bash_utils/get_bash_file_contents.sh @@ -21,7 +21,7 @@ function get_bash_file_contents() { local valid_args=( \ "fp" \ - "output_varname_contents" \ + "outvarname_contents" \ ) process_args valid_args "$@" print_input_args "valid_args" @@ -63,7 +63,7 @@ read was not specified in the call to this function: # # Set output variables. # - printf -v ${output_varname_contents} "${contents}" + printf -v ${outvarname_contents} "%s" "${contents}" { restore_shell_opts; } > /dev/null 2>&1 diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh index 7476de7792..c55a392295 100644 --- a/ush/check_expt_config_vars.sh +++ b/ush/check_expt_config_vars.sh @@ -47,10 +47,10 @@ function check_expt_config_vars() { # configuration file. # get_bash_file_contents fp="${default_config_fp}" \ - output_varname_contents="var_list_default" + outvarname_contents="var_list_default" get_bash_file_contents fp="${config_fp}" \ - output_varname_contents="var_list_user" + outvarname_contents="var_list_user" # # Loop through each line/variable in var_list_user. For each line, # extract the the name of the variable that is being set (say VAR) and diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index d304d203a4..966bf343d2 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -74,12 +74,15 @@ RUN_ENVIR="nco" # for supported machines. # # BUILD_MOD_FN: -# Name of alternative build module file to use if using an -# unsupported platform. Is set automatically for supported machines. +# Name of build module file to load before running a workflow task. If +# this is not specified by the user in the experiment configuration file +# (EXPT_CONFIG_FN), it will be set automatically for supported machines. # # WFLOW_MOD_FN: -# Name of alternative workflow module file to use if using an -# unsupported platform. Is set automatically for supported machines. +# Name of workflow module file to load before (re)launching the experiment's +# ROCOTO workflow (using the "rocotorun" command). If this is not specified +# by the user in the experiment configuration file (EXPT_CONFIG_FN), it +# will be set automatically for supported machines. # # SCHED: # The job scheduler to use (e.g. slurm). Set this to an empty string in @@ -307,6 +310,10 @@ DOT_OR_USCORE="_" # EXPT_CONFIG_FN: # Name of the user-specified configuration file for the forecast experiment. # +# CONSTANTS_FN: +# Name of the file containing definitions of various mathematical, physical, +# and SRW App contants. +# # RGNL_GRID_NML_FN: # Name of file containing the namelist settings for the code that generates # a "ESGgrid" type of regional grid. @@ -388,6 +395,7 @@ DOT_OR_USCORE="_" #----------------------------------------------------------------------- # EXPT_CONFIG_FN="config.sh" +CONSTANTS_FN="constants.sh" RGNL_GRID_NML_FN="regional_grid.nml" diff --git a/ush/constants.sh b/ush/constants.sh index 0d68019aed..26f4cb1b32 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -10,16 +10,56 @@ pi_geom="3.14159265358979323846264338327" # Degrees per radian. -degs_per_radian=$( bc -l <<< "360.0/(2.0*$pi_geom)" ) +degs_per_radian=$( bc -l <<< "360.0/(2.0*${pi_geom})" ) # Radius of the Earth in meters. radius_Earth="6371200.0" - # #----------------------------------------------------------------------- # -# Other. +# Valid values that a user may set a boolean variable to (e.g. in the +# SRW App's experiment configuration file). # #----------------------------------------------------------------------- # valid_vals_BOOLEAN=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +# +#----------------------------------------------------------------------- +# +# Any regional model must be supplied lateral boundary conditions (in +# addition to initial conditions) to be able to perform a forecast. In +# the FV3-LAM model, these boundary conditions (BCs) are supplied using +# a "halo" of grid cells around the regional domain that extend beyond +# the boundary of the domain. The model is formulated such that along +# with files containing these BCs, it needs as input the following files +# (in NetCDF format): +# +# 1) A grid file that includes a halo of 3 cells beyond the boundary of +# the domain. +# 2) A grid file that includes a halo of 4 cells beyond the boundary of +# the domain. +# 3) A (filtered) orography file without a halo, i.e. a halo of width +# 0 cells. +# 4) A (filtered) orography file that includes a halo of 4 cells beyond +# the boundary of the domain. +# +# Note that the regional grid is referred to as "tile 7" in the code. +# We will let: +# +# * NH0 denote the width (in units of number of cells on tile 7) of +# the 0-cell-wide halo, i.e. NH0 = 0; +# +# * NH3 denote the width (in units of number of cells on tile 7) of +# the 3-cell-wide halo, i.e. NH3 = 3; and +# +# * NH4 denote the width (in units of number of cells on tile 7) of +# the 4-cell-wide halo, i.e. NH4 = 4. +# +# We define these variables next. +# +#----------------------------------------------------------------------- +# +NH0=0 +NH3=3 +NH4=4 + diff --git a/ush/get_crontab_contents.sh b/ush/get_crontab_contents.sh index 182cf19365..a7a00854ef 100644 --- a/ush/get_crontab_contents.sh +++ b/ush/get_crontab_contents.sh @@ -42,7 +42,7 @@ function get_crontab_contents() { # source $USHDIR/constants.sh check_var_valid_value "called_from_cron" "valid_vals_BOOLEAN" - called_from_cron=$( boolify ${called_from_cron} ) + called_from_cron=$(boolify "${called_from_cron}") if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then __crontab_cmd__="" diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 9184cb0e0c..8a4dbe6031 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -121,7 +121,7 @@ print_input_args "valid_args" # called_from_cron=${called_from_cron:-"FALSE"} check_var_valid_value "called_from_cron" "valid_vals_BOOLEAN" -called_from_cron=$(boolify ${called_from_cron}) +called_from_cron=$(boolify "${called_from_cron}") # #----------------------------------------------------------------------- # @@ -151,7 +151,7 @@ expt_name="${EXPT_SUBDIR}" # module use "${SR_WX_APP_TOP_DIR}/modulefiles" module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || print_err_msg_exit "\ -Loading platform-specific module file (WFLOW_MOD_FN) for the workflow +Loading of platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: WFLOW_MOD_FN = \"${WFLOW_MOD_FN}\"" # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 7a9547fe03..73c7c7eed2 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -95,8 +95,8 @@ machine=$(echo_lowercase $MACHINE) source "${SR_WX_APP_TOP_DIR}/etc/lmod-setup.sh" module use "${SR_WX_APP_TOP_DIR}/modulefiles" module load "${BUILD_MOD_FN}" || print_err_msg_exit "\ -Sourcing platform- and compiler-specific module file (BUILD_MOD_FN) for the -workflow task specified by task_name failed: +Loading of platform- and compiler-specific module file (BUILD_MOD_FN) +for the workflow task specified by task_name failed: task_name = \"${task_name}\" BUILD_MOD_FN = \"${BUILD_MOD_FN}\"" # diff --git a/ush/set_gridparams_ESGgrid.sh b/ush/set_gridparams_ESGgrid.sh index 41efbfe582..4b81377713 100644 --- a/ush/set_gridparams_ESGgrid.sh +++ b/ush/set_gridparams_ESGgrid.sh @@ -40,6 +40,23 @@ function set_gridparams_ESGgrid() { # #----------------------------------------------------------------------- # +# Set directories. +# +#----------------------------------------------------------------------- +# + local homerrfs=${scrfunc_dir%/*} + local ushdir="$homerrfs/ush" +# +#----------------------------------------------------------------------- +# +# Source the file containing various mathematical, physical, etc constants. +# +#----------------------------------------------------------------------- +# + . $ushdir/constants.sh +# +#----------------------------------------------------------------------- +# # Specify the set of valid argument names for this script/function. # Then process the arguments provided to this script/function (which # should consist of a set of name-value pairs of the form arg1="value1", @@ -48,25 +65,25 @@ function set_gridparams_ESGgrid() { #----------------------------------------------------------------------- # local valid_args=( \ -"lon_ctr" \ -"lat_ctr" \ -"nx" \ -"ny" \ -"halo_width" \ -"delx" \ -"dely" \ -"pazi" \ -"output_varname_lon_ctr" \ -"output_varname_lat_ctr" \ -"output_varname_nx" \ -"output_varname_ny" \ -"output_varname_pazi" \ -"output_varname_halo_width" \ -"output_varname_stretch_factor" \ -"output_varname_del_angle_x_sg" \ -"output_varname_del_angle_y_sg" \ -"output_varname_neg_nx_of_dom_with_wide_halo" \ -"output_varname_neg_ny_of_dom_with_wide_halo" \ + "lon_ctr" \ + "lat_ctr" \ + "nx" \ + "ny" \ + "halo_width" \ + "delx" \ + "dely" \ + "pazi" \ + "outvarname_lon_ctr" \ + "outvarname_lat_ctr" \ + "outvarname_nx" \ + "outvarname_ny" \ + "outvarname_pazi" \ + "outvarname_halo_width" \ + "outvarname_stretch_factor" \ + "outvarname_del_angle_x_sg" \ + "outvarname_del_angle_y_sg" \ + "outvarname_neg_nx_of_dom_with_wide_halo" \ + "outvarname_neg_ny_of_dom_with_wide_halo" \ ) process_args valid_args "$@" # @@ -78,15 +95,7 @@ function set_gridparams_ESGgrid() { # #----------------------------------------------------------------------- # - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Source the file containing various mathematical, physical, etc constants. -# -#----------------------------------------------------------------------- -# - . ${USHDIR}/constants.sh +# print_input_args valid_args # #----------------------------------------------------------------------- # @@ -149,17 +158,49 @@ function set_gridparams_ESGgrid() { # #----------------------------------------------------------------------- # - eval ${output_varname_lon_ctr}="${lon_ctr}" - eval ${output_varname_lat_ctr}="${lat_ctr}" - eval ${output_varname_nx}="${nx}" - eval ${output_varname_ny}="${ny}" - eval ${output_varname_halo_width}="${halo_width}" - eval ${output_varname_stretch_factor}="${stretch_factor}" - eval ${output_varname_pazi}="${pazi}" - eval ${output_varname_del_angle_x_sg}="${del_angle_x_sg}" - eval ${output_varname_del_angle_y_sg}="${del_angle_y_sg}" - eval ${output_varname_neg_nx_of_dom_with_wide_halo}="${neg_nx_of_dom_with_wide_halo}" - eval ${output_varname_neg_ny_of_dom_with_wide_halo}="${neg_ny_of_dom_with_wide_halo}" + if [ ! -z "${outvarname_lon_ctr}" ]; then + printf -v ${outvarname_lon_ctr} "%s" "${lon_ctr}" + fi + + if [ ! -z "${outvarname_lat_ctr}" ]; then + printf -v ${outvarname_lat_ctr} "%s" "${lat_ctr}" + fi + + if [ ! -z "${outvarname_nx}" ]; then + printf -v ${outvarname_nx} "%s" "${nx}" + fi + + if [ ! -z "${outvarname_ny}" ]; then + printf -v ${outvarname_ny} "%s" "${ny}" + fi + + if [ ! -z "${outvarname_halo_width}" ]; then + printf -v ${outvarname_halo_width} "%s" "${halo_width}" + fi + + if [ ! -z "${outvarname_stretch_factor}" ]; then + printf -v ${outvarname_stretch_factor} "%s" "${stretch_factor}" + fi + + if [ ! -z "${outvarname_pazi}" ]; then + printf -v ${outvarname_pazi} "%s" "${pazi}" + fi + + if [ ! -z "${outvarname_del_angle_x_sg}" ]; then + printf -v ${outvarname_del_angle_x_sg} "%s" "${del_angle_x_sg}" + fi + + if [ ! -z "${outvarname_del_angle_y_sg}" ]; then + printf -v ${outvarname_del_angle_y_sg} "%s" "${del_angle_y_sg}" + fi + + if [ ! -z "${outvarname_neg_nx_of_dom_with_wide_halo}" ]; then + printf -v ${outvarname_neg_nx_of_dom_with_wide_halo} "%s" "${neg_nx_of_dom_with_wide_halo}" + fi + + if [ ! -z "${outvarname_neg_ny_of_dom_with_wide_halo}" ]; then + printf -v ${outvarname_neg_ny_of_dom_with_wide_halo} "%s" "${neg_ny_of_dom_with_wide_halo}" + fi # #----------------------------------------------------------------------- # diff --git a/ush/set_gridparams_GFDLgrid.sh b/ush/set_gridparams_GFDLgrid.sh index a76b9a58a0..68b6937be1 100644 --- a/ush/set_gridparams_GFDLgrid.sh +++ b/ush/set_gridparams_GFDLgrid.sh @@ -40,6 +40,23 @@ function set_gridparams_GFDLgrid() { # #----------------------------------------------------------------------- # +# Set directories. +# +#----------------------------------------------------------------------- +# + local homerrfs=${scrfunc_dir%/*} + local ushdir="$homerrfs/ush" +# +#----------------------------------------------------------------------- +# +# Source the file containing various mathematical, physical, etc constants. +# +#----------------------------------------------------------------------- +# + . $ushdir/constants.sh +# +#----------------------------------------------------------------------- +# # Specify the set of valid argument names for this script/function. # Then process the arguments provided to this script/function (which # should consist of a set of name-value pairs of the form arg1="value1", @@ -48,25 +65,26 @@ function set_gridparams_GFDLgrid() { #----------------------------------------------------------------------- # local valid_args=( \ -"lon_of_t6_ctr" \ -"lat_of_t6_ctr" \ -"res_of_t6g" \ -"stretch_factor" \ -"refine_ratio_t6g_to_t7g" \ -"istart_of_t7_on_t6g" \ -"iend_of_t7_on_t6g" \ -"jstart_of_t7_on_t6g" \ -"jend_of_t7_on_t6g" \ -"output_varname_lon_of_t7_ctr" \ -"output_varname_lat_of_t7_ctr" \ -"output_varname_nx_of_t7_on_t7g" \ -"output_varname_ny_of_t7_on_t7g" \ -"output_varname_halo_width_on_t7g" \ -"output_varname_stretch_factor" \ -"output_varname_istart_of_t7_with_halo_on_t6sg" \ -"output_varname_iend_of_t7_with_halo_on_t6sg" \ -"output_varname_jstart_of_t7_with_halo_on_t6sg" \ -"output_varname_jend_of_t7_with_halo_on_t6sg" \ + "lon_of_t6_ctr" \ + "lat_of_t6_ctr" \ + "res_of_t6g" \ + "stretch_factor" \ + "refine_ratio_t6g_to_t7g" \ + "istart_of_t7_on_t6g" \ + "iend_of_t7_on_t6g" \ + "jstart_of_t7_on_t6g" \ + "jend_of_t7_on_t6g" \ + "verbose" \ + "outvarname_lon_of_t7_ctr" \ + "outvarname_lat_of_t7_ctr" \ + "outvarname_nx_of_t7_on_t7g" \ + "outvarname_ny_of_t7_on_t7g" \ + "outvarname_halo_width_on_t7g" \ + "outvarname_stretch_factor" \ + "outvarname_istart_of_t7_with_halo_on_t6sg" \ + "outvarname_iend_of_t7_with_halo_on_t6sg" \ + "outvarname_jstart_of_t7_with_halo_on_t6sg" \ + "outvarname_jend_of_t7_with_halo_on_t6sg" \ ) process_args valid_args "$@" # @@ -140,7 +158,6 @@ function set_gridparams_GFDLgrid() { # This if-statement can hopefully be removed once EMC agrees to make their # GFDLgrid type grids (tile 7) symmetric about tile 6. -if [ "${RUN_ENVIR}" != "nco" ]; then if [ ${num_left_margin_cells_on_t6g} -ne ${num_right_margin_cells_on_t6g} ]; then print_err_msg_exit "\ In order for tile 7 to be centered in the x direction on tile 6, the x- @@ -159,14 +176,12 @@ ven by: nx_of_t6_on_t6g = ${nx_of_t6_on_t6g} Please reset istart_of_t7_on_t6g and iend_of_t7_on_t6g and rerun." fi -fi num_bot_margin_cells_on_t6g=$(( jstart_of_t7_on_t6g - 1 )) num_top_margin_cells_on_t6g=$(( ny_of_t6_on_t6g - jend_of_t7_on_t6g )) # This if-statement can hopefully be removed once EMC agrees to make their # GFDLgrid type grids (tile 7) symmetric about tile 6. -if [ "${RUN_ENVIR}" != "nco" ]; then if [ ${num_bot_margin_cells_on_t6g} -ne ${num_top_margin_cells_on_t6g} ]; then print_err_msg_exit "\ In order for tile 7 to be centered in the y direction on tile 6, the y- @@ -185,7 +200,6 @@ ven by: ny_of_t6_on_t6g = ${ny_of_t6_on_t6g} Please reset jstart_of_t7_on_t6g and jend_of_t7_on_t6g and rerun." fi -fi lon_of_t7_ctr="${lon_of_t6_ctr}" lat_of_t7_ctr="${lat_of_t6_ctr}" @@ -386,7 +400,7 @@ fi # #----------------------------------------------------------------------- # - print_info_msg "$VERBOSE" " + print_info_msg "$verbose" " Original values of the halo width on the tile 6 supergrid and on the tile 7 grid are: halo_width_on_t6sg = ${halo_width_on_t6sg} @@ -396,7 +410,7 @@ tile 7 grid are: halo_width_on_t6g=$(( halo_width_on_t6sg/2 )) halo_width_on_t7g=$(( halo_width_on_t6g*refine_ratio_t6g_to_t7g )) - print_info_msg "$VERBOSE" " + print_info_msg "$verbose" " Values of the halo width on the tile 6 supergrid and on the tile 7 grid AFTER adjustments are: halo_width_on_t6sg = ${halo_width_on_t6sg} @@ -428,7 +442,7 @@ AFTER adjustments are: prime_factors_nx_of_t7_on_t7g=$( factor ${nx_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) prime_factors_ny_of_t7_on_t7g=$( factor ${ny_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) - print_info_msg "$VERBOSE" " + print_info_msg "$verbose" " The number of cells in the two horizontal directions (x and y) on the parent tile's (tile 6) grid and supergrid are: nx_of_t6_on_t6g = ${nx_of_t6_on_t6g} @@ -489,12 +503,12 @@ determining an MPI task layout): ny_of_t7_with_halo_on_t6g=$(( ny_of_t7_with_halo_on_t6sg/2 )) ny_of_t7_with_halo_on_t7g=$(( ny_of_t7_with_halo_on_t6g*refine_ratio_t6g_to_t7g )) - print_info_msg "$VERBOSE" " + print_info_msg "$verbose" " nx_of_t7_with_halo_on_t7g = ${nx_of_t7_with_halo_on_t7g} \ (istart_of_t7_with_halo_on_t6sg = ${istart_of_t7_with_halo_on_t6sg}, \ iend_of_t7_with_halo_on_t6sg = ${iend_of_t7_with_halo_on_t6sg})" - print_info_msg "$VERBOSE" " + print_info_msg "$verbose" " ny_of_t7_with_halo_on_t7g = ${ny_of_t7_with_halo_on_t7g} \ (jstart_of_t7_with_halo_on_t6sg = ${jstart_of_t7_with_halo_on_t6sg}, \ jend_of_t7_with_halo_on_t6sg = ${jend_of_t7_with_halo_on_t6sg})" @@ -505,16 +519,45 @@ jend_of_t7_with_halo_on_t6sg = ${jend_of_t7_with_halo_on_t6sg})" # #----------------------------------------------------------------------- # - eval ${output_varname_lon_of_t7_ctr}="${lon_of_t7_ctr}" - eval ${output_varname_lat_of_t7_ctr}="${lat_of_t7_ctr}" - eval ${output_varname_nx_of_t7_on_t7g}="${nx_of_t7_on_t7g}" - eval ${output_varname_ny_of_t7_on_t7g}="${ny_of_t7_on_t7g}" - eval ${output_varname_halo_width_on_t7g}="${halo_width_on_t7g}" - eval ${output_varname_stretch_factor}="${stretch_factor}" - eval ${output_varname_istart_of_t7_with_halo_on_t6sg}="${istart_of_t7_with_halo_on_t6sg}" - eval ${output_varname_iend_of_t7_with_halo_on_t6sg}="${iend_of_t7_with_halo_on_t6sg}" - eval ${output_varname_jstart_of_t7_with_halo_on_t6sg}="${jstart_of_t7_with_halo_on_t6sg}" - eval ${output_varname_jend_of_t7_with_halo_on_t6sg}="${jend_of_t7_with_halo_on_t6sg}" + if [ ! -z "${outvarname_lon_of_t7_ctr}" ]; then + printf -v ${outvarname_lon_of_t7_ctr} "%s" "${lon_of_t7_ctr}" + fi + + if [ ! -z "${outvarname_lat_of_t7_ctr}" ]; then + printf -v ${outvarname_lat_of_t7_ctr} "%s" "${lat_of_t7_ctr}" + fi + + if [ ! -z "${outvarname_nx_of_t7_on_t7g}" ]; then + printf -v ${outvarname_nx_of_t7_on_t7g} "%s" "${nx_of_t7_on_t7g}" + fi + + if [ ! -z "${outvarname_ny_of_t7_on_t7g}" ]; then + printf -v ${outvarname_ny_of_t7_on_t7g} "%s" "${ny_of_t7_on_t7g}" + fi + + if [ ! -z "${outvarname_halo_width_on_t7g}" ]; then + printf -v ${outvarname_halo_width_on_t7g} "%s" "${halo_width_on_t7g}" + fi + + if [ ! -z "${outvarname_stretch_factor}" ]; then + printf -v ${outvarname_stretch_factor} "%s" "${stretch_factor}" + fi + + if [ ! -z "${outvarname_istart_of_t7_with_halo_on_t6sg}" ]; then + printf -v ${outvarname_istart_of_t7_with_halo_on_t6sg} "%s" "${istart_of_t7_with_halo_on_t6sg}" + fi + + if [ ! -z "${outvarname_iend_of_t7_with_halo_on_t6sg}" ]; then + printf -v ${outvarname_iend_of_t7_with_halo_on_t6sg} "%s" "${iend_of_t7_with_halo_on_t6sg}" + fi + + if [ ! -z "${outvarname_jstart_of_t7_with_halo_on_t6sg}" ]; then + printf -v ${outvarname_jstart_of_t7_with_halo_on_t6sg} "%s" "${jstart_of_t7_with_halo_on_t6sg}" + fi + + if [ ! -z "${outvarname_jend_of_t7_with_halo_on_t6sg}" ]; then + printf -v ${outvarname_jend_of_t7_with_halo_on_t6sg} "%s" "${jend_of_t7_with_halo_on_t6sg}" + fi # #----------------------------------------------------------------------- # diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index fd4ef530bb..71c3d12a7e 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -15,7 +15,7 @@ function set_predef_grid_params() { # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -25,9 +25,9 @@ function set_predef_grid_params() { # #----------------------------------------------------------------------- # -local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) + local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # @@ -35,7 +35,16 @@ local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -local func_name="${FUNCNAME[0]}" + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Set directories. +# +#----------------------------------------------------------------------- +# + local homerrfs=${scrfunc_dir%/*} + local ushdir="$homerrfs/ush" # #----------------------------------------------------------------------- # @@ -43,13 +52,142 @@ local func_name="${FUNCNAME[0]}" # #----------------------------------------------------------------------- # -. ${USHDIR}/constants.sh + . $ushdir/constants.sh +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "predef_grid_name" \ + "dt_atmos" \ + "layout_x" \ + "layout_y" \ + "blocksize" \ + "quilting" \ + "outvarname_grid_gen_method" \ + "outvarname_esggrid_lon_ctr" \ + "outvarname_esggrid_lat_ctr" \ + "outvarname_esggrid_delx" \ + "outvarname_esggrid_dely" \ + "outvarname_esggrid_nx" \ + "outvarname_esggrid_ny" \ + "outvarname_esggrid_pazi" \ + "outvarname_esggrid_wide_halo_width" \ + "outvarname_gfdlgrid_lon_t6_ctr" \ + "outvarname_gfdlgrid_lat_t6_ctr" \ + "outvarname_gfdlgrid_stretch_fac" \ + "outvarname_gfdlgrid_res" \ + "outvarname_gfdlgrid_refine_ratio" \ + "outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g" \ + "outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g" \ + "outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g" \ + "outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g" \ + "outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames" \ + "outvarname_dt_atmos" \ + "outvarname_layout_x" \ + "outvarname_layout_y" \ + "outvarname_blocksize" \ + "outvarname_wrtcmp_write_groups" \ + "outvarname_wrtcmp_write_tasks_per_group" \ + "outvarname_wrtcmp_output_grid" \ + "outvarname_wrtcmp_cen_lon" \ + "outvarname_wrtcmp_cen_lat" \ + "outvarname_wrtcmp_stdlat1" \ + "outvarname_wrtcmp_stdlat2" \ + "outvarname_wrtcmp_nx" \ + "outvarname_wrtcmp_ny" \ + "outvarname_wrtcmp_lon_lwr_left" \ + "outvarname_wrtcmp_lat_lwr_left" \ + "outvarname_wrtcmp_lon_upr_rght" \ + "outvarname_wrtcmp_lat_upr_rght" \ + "outvarname_wrtcmp_dx" \ + "outvarname_wrtcmp_dy" \ + "outvarname_wrtcmp_dlon" \ + "outvarname_wrtcmp_dlat" \ + ) + process_args "valid_args" "$@" +# +#----------------------------------------------------------------------- +# +# Declare and initialize local variables. +# +#----------------------------------------------------------------------- +# + local __grid_gen_method__="" \ + __esggrid_lon_ctr__="" \ + __esggrid_lat_ctr__="" \ + __esggrid_delx__="" \ + __esggrid_dely__="" \ + __esggrid_nx__="" \ + __esggrid_ny__="" \ + __esggrid_pazi__="" \ + __esggrid_wide_halo_width__="" \ + __gfdlgrid_lon_t6_ctr__="" \ + __gfdlgrid_lat_t6_ctr__="" \ + __gfdlgrid_stretch_fac__="" \ + __gfdlgrid_res__="" \ + __gfdlgrid_refine_ratio__="" \ + __gfdlgrid_istart_of_rgnl_dom_on_t6g__="" \ + __gfdlgrid_iend_of_rgnl_dom_on_t6g__="" \ + __gfdlgrid_jstart_of_rgnl_dom_on_t6g__="" \ + __gfdlgrid_jend_of_rgnl_dom_on_t6g__="" \ + __gfdlgrid_use_gfdlgrid_res_in_filenames__="" \ + __dt_atmos__="" \ + __layout_x__="" \ + __layout_y__="" \ + __blocksize__="" \ + __wrtcmp_write_groups__="" \ + __wrtcmp_write_tasks_per_group__="" \ + __wrtcmp_output_grid__="" \ + __wrtcmp_cen_lon__="" \ + __wrtcmp_cen_lat__="" \ + __wrtcmp_stdlat1__="" \ + __wrtcmp_stdlat2__="" \ + __wrtcmp_nx__="" \ + __wrtcmp_ny__="" \ + __wrtcmp_lon_lwr_left__="" \ + __wrtcmp_lat_lwr_left__="" \ + __wrtcmp_lon_upr_rght__="" \ + __wrtcmp_lat_upr_rght__="" \ + __wrtcmp_dx__="" \ + __wrtcmp_dy__="" \ + __wrtcmp_dlon__="" \ + __wrtcmp_dlat__="" \ + num_margin_cells_T6_left="" \ + num_margin_cells_T6_right="" \ + num_margin_cells_T6_bottom="" \ + num_margin_cells_T6_top="" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +# print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Make sure that the input argument "quilting" is set to a valid value. +# +#----------------------------------------------------------------------- +# + check_var_valid_value "quilting" "valid_vals_BOOLEAN" + quilting=$(boolify "$quilting") # #----------------------------------------------------------------------- # # Set grid and other parameters according to the value of the predefined -# domain (PREDEF_GRID_NAME). Note that the code will enter this script -# only if PREDEF_GRID_NAME has a valid (and non-empty) value. +# domain (predef_grid_name). Note that the code will enter this script +# only if predef_grid_name has a valid (and non-empty) value. # #################### # The following comments need to be updated: @@ -57,7 +195,7 @@ local func_name="${FUNCNAME[0]}" # # 1) Reset the experiment title (expt_title). # 2) Reset the grid parameters. -# 3) If the write component is to be used (i.e. QUILTING is set to +# 3) If the write component is to be used (i.e. "quilting" is set to # "TRUE") and the variable WRTCMP_PARAMS_TMPL_FN containing the name # of the write-component template file is unset or empty, set that # filename variable to the appropriate preexisting template file. @@ -91,7 +229,7 @@ local func_name="${FUNCNAME[0]}" # #----------------------------------------------------------------------- # -case ${PREDEF_GRID_NAME} in + case "${predef_grid_name}" in # #----------------------------------------------------------------------- # @@ -99,91 +237,92 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_CONUS_25km") + "RRFS_CONUS_25km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="25000.0" - ESGgrid_DELY="25000.0" + __esggrid_delx__="25000.0" + __esggrid_dely__="25000.0" - ESGgrid_NX="219" - ESGgrid_NY="131" + __esggrid_nx__="219" + __esggrid_ny__="131" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-40}" + __dt_atmos__="${dt_atmos:-40}" - LAYOUT_X="${LAYOUT_X:-5}" - LAYOUT_Y="${LAYOUT_Y:-2}" - BLOCKSIZE="${BLOCKSIZE:-40}" + __layout_x__="${layout_x:-5}" + __layout_y__="${layout_y:-2}" + __blocksize__="${blocksize:-40}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="2" - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="217" - WRTCMP_ny="128" - WRTCMP_lon_lwr_left="-122.719528" - WRTCMP_lat_lwr_left="21.138123" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="2" + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="217" + __wrtcmp_ny__="128" + __wrtcmp_lon_lwr_left__="-122.719528" + __wrtcmp_lat_lwr_left__="21.138123" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # -# The RRFS CONUS domain with ~25km cells that can be initialized from the HRRR. +# The RRFS CONUS domain with ~25km cells that can be initialized from +# the HRRR. # #----------------------------------------------------------------------- # -"RRFS_CONUScompact_25km") + "RRFS_CONUScompact_25km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="25000.0" - ESGgrid_DELY="25000.0" + __esggrid_delx__="25000.0" + __esggrid_dely__="25000.0" - ESGgrid_NX="202" - ESGgrid_NY="116" + __esggrid_nx__="202" + __esggrid_ny__="116" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-40}" + __dt_atmos__="${dt_atmos:-40}" - LAYOUT_X="${LAYOUT_X:-5}" - LAYOUT_Y="${LAYOUT_Y:-2}" - BLOCKSIZE="${BLOCKSIZE:-40}" + __layout_x__="${layout_x:-5}" + __layout_y__="${layout_y:-2}" + __blocksize__="${blocksize:-40}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="2" - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="199" - WRTCMP_ny="111" - WRTCMP_lon_lwr_left="-121.23349066" - WRTCMP_lat_lwr_left="23.41731593" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="2" + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="199" + __wrtcmp_ny__="111" + __wrtcmp_lon_lwr_left__="-121.23349066" + __wrtcmp_lat_lwr_left__="23.41731593" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -191,45 +330,45 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_CONUS_13km") + "RRFS_CONUS_13km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="13000.0" - ESGgrid_DELY="13000.0" + __esggrid_delx__="13000.0" + __esggrid_dely__="13000.0" - ESGgrid_NX="420" - ESGgrid_NY="252" + __esggrid_nx__="420" + __esggrid_ny__="252" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-45}" + __dt_atmos__="${dt_atmos:-45}" - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-10}" - BLOCKSIZE="${BLOCKSIZE:-32}" + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-10}" + __blocksize__="${blocksize:-32}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="416" - WRTCMP_ny="245" - WRTCMP_lon_lwr_left="-122.719528" - WRTCMP_lat_lwr_left="21.138123" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="416" + __wrtcmp_ny__="245" + __wrtcmp_lon_lwr_left__="-122.719528" + __wrtcmp_lat_lwr_left__="21.138123" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -237,45 +376,45 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_CONUScompact_13km") + "RRFS_CONUScompact_13km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="13000.0" - ESGgrid_DELY="13000.0" + __esggrid_delx__="13000.0" + __esggrid_dely__="13000.0" - ESGgrid_NX="396" - ESGgrid_NY="232" + __esggrid_nx__="396" + __esggrid_ny__="232" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-45}" + __dt_atmos__="${dt_atmos:-45}" - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-10}" - BLOCKSIZE="${BLOCKSIZE:-32}" + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-10}" + __blocksize__="${blocksize:-32}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="393" - WRTCMP_ny="225" - WRTCMP_lon_lwr_left="-121.70231097" - WRTCMP_lat_lwr_left="22.57417972" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="393" + __wrtcmp_ny__="225" + __wrtcmp_lon_lwr_left__="-121.70231097" + __wrtcmp_lat_lwr_left__="22.57417972" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -283,91 +422,92 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_CONUS_3km") + "RRFS_CONUS_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="1820" - ESGgrid_NY="1092" + __esggrid_nx__="1820" + __esggrid_ny__="1092" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-36}" + __dt_atmos__="${dt_atmos:-36}" - LAYOUT_X="${LAYOUT_X:-28}" - LAYOUT_Y="${LAYOUT_Y:-28}" - BLOCKSIZE="${BLOCKSIZE:-29}" + __layout_x__="${layout_x:-28}" + __layout_y__="${layout_y:-28}" + __blocksize__="${blocksize:-29}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="1799" - WRTCMP_ny="1059" - WRTCMP_lon_lwr_left="-122.719528" - WRTCMP_lat_lwr_left="21.138123" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="1799" + __wrtcmp_ny__="1059" + __wrtcmp_lon_lwr_left__="-122.719528" + __wrtcmp_lat_lwr_left__="21.138123" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # -# The RRFS CONUS domain with ~3km cells that can be initialized from the HRRR. +# The RRFS CONUS domain with ~3km cells that can be initialized from +# the HRRR. # #----------------------------------------------------------------------- # -"RRFS_CONUScompact_3km") + "RRFS_CONUScompact_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="1748" - ESGgrid_NY="1038" + __esggrid_nx__="1748" + __esggrid_ny__="1038" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-40}" + __dt_atmos__="${dt_atmos:-40}" - LAYOUT_X="${LAYOUT_X:-30}" - LAYOUT_Y="${LAYOUT_Y:-16}" - BLOCKSIZE="${BLOCKSIZE:-32}" + __layout_x__="${layout_x:-30}" + __layout_y__="${layout_y:-16}" + __blocksize__="${blocksize:-32}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="1746" - WRTCMP_ny="1014" - WRTCMP_lon_lwr_left="-122.17364391" - WRTCMP_lat_lwr_left="21.88588562" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="1746" + __wrtcmp_ny__="1014" + __wrtcmp_lon_lwr_left__="-122.17364391" + __wrtcmp_lat_lwr_left__="21.88588562" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -375,45 +515,45 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_SUBCONUS_3km") + "RRFS_SUBCONUS_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="35.0" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="35.0" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="840" - ESGgrid_NY="600" + __esggrid_nx__="840" + __esggrid_ny__="600" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-40}" + __dt_atmos__="${dt_atmos:-40}" - LAYOUT_X="${LAYOUT_X:-30}" - LAYOUT_Y="${LAYOUT_Y:-24}" - BLOCKSIZE="${BLOCKSIZE:-35}" + __layout_x__="${layout_x:-30}" + __layout_y__="${layout_y:-24}" + __blocksize__="${blocksize:-35}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="837" - WRTCMP_ny="595" - WRTCMP_lon_lwr_left="-109.97410429" - WRTCMP_lat_lwr_left="26.31459843" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="837" + __wrtcmp_ny__="595" + __wrtcmp_lon_lwr_left__="-109.97410429" + __wrtcmp_lat_lwr_left__="26.31459843" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -423,45 +563,45 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"SUBCONUS_Ind_3km") + "SUBCONUS_Ind_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-86.16" - ESGgrid_LAT_CTR="39.77" + __esggrid_lon_ctr__="-86.16" + __esggrid_lat_ctr__="39.77" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="200" - ESGgrid_NY="200" + __esggrid_nx__="200" + __esggrid_ny__="200" - ESGgrid_PAZI="0.0" - - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_pazi__="0.0" - DT_ATMOS="${DT_ATMOS:-40}" + __esggrid_wide_halo_width__="6" - LAYOUT_X="${LAYOUT_X:-5}" - LAYOUT_Y="${LAYOUT_Y:-5}" - BLOCKSIZE="${BLOCKSIZE:-40}" + __dt_atmos__="${dt_atmos:-40}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="197" - WRTCMP_ny="197" - WRTCMP_lon_lwr_left="-89.47120417" - WRTCMP_lat_lwr_left="37.07809642" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + __layout_x__="${layout_x:-5}" + __layout_y__="${layout_y:-5}" + __blocksize__="${blocksize:-40}" + + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="197" + __wrtcmp_ny__="197" + __wrtcmp_lon_lwr_left__="-89.47120417" + __wrtcmp_lat_lwr_left__="37.07809642" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -472,70 +612,71 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_AK_13km") + "RRFS_AK_13km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-161.5" - ESGgrid_LAT_CTR="63.0" + __esggrid_lon_ctr__="-161.5" + __esggrid_lat_ctr__="63.0" - ESGgrid_DELX="13000.0" - ESGgrid_DELY="13000.0" + __esggrid_delx__="13000.0" + __esggrid_dely__="13000.0" - ESGgrid_NX="320" - ESGgrid_NY="240" + __esggrid_nx__="320" + __esggrid_ny__="240" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" -# DT_ATMOS="${DT_ATMOS:-50}" - DT_ATMOS="${DT_ATMOS:-10}" +# __dt_atmos__="${dt_atmos:-50}" + __dt_atmos__="${dt_atmos:-10}" - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-12}" - BLOCKSIZE="${BLOCKSIZE:-40}" + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-12}" + __blocksize__="${blocksize:-40}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" -# The following work. They were obtained using the NCL scripts but only -# after manually modifying the longitutes of two of the 4 corners of the -# domain to add 360.0 to them. Need to automate that procedure. - WRTCMP_nx="318" - WRTCMP_ny="234" -# WRTCMP_lon_lwr_left="-187.76660836" - WRTCMP_lon_lwr_left="172.23339164" - WRTCMP_lat_lwr_left="45.77691870" +# The following works. The numbers were obtained using the NCL scripts +# but only after manually modifying the longitutes of two of the four +# corners of the domain to add 360.0 to them. Need to automate that +# procedure. + __wrtcmp_nx__="318" + __wrtcmp_ny__="234" +# __wrtcmp_lon_lwr_left__="-187.76660836" + __wrtcmp_lon_lwr_left__="172.23339164" + __wrtcmp_lat_lwr_left__="45.77691870" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi # The following rotated_latlon coordinate system parameters were obtained # using the NCL code and work. -# if [ "$QUILTING" = "TRUE" ]; then -# WRTCMP_write_groups="1" -# WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) -# WRTCMP_output_grid="rotated_latlon" -# WRTCMP_cen_lon="${ESGgrid_LON_CTR}" -# WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" -# WRTCMP_lon_lwr_left="-18.47206579" -# WRTCMP_lat_lwr_left="-13.56176982" -# WRTCMP_lon_upr_rght="18.47206579" -# WRTCMP_lat_upr_rght="13.56176982" -## WRTCMP_dlon="0.11691181" -## WRTCMP_dlat="0.11691181" -# WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) -# WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) -# fi - ;; +# if [ "$quilting" = "TRUE" ]; then +# __wrtcmp_write_groups__="1" +# __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) +# __wrtcmp_output_grid__="rotated_latlon" +# __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" +# __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" +# __wrtcmp_lon_lwr_left__="-18.47206579" +# __wrtcmp_lat_lwr_left__="-13.56176982" +# __wrtcmp_lon_upr_rght__="18.47206579" +# __wrtcmp_lat_upr_rght__="13.56176982" +## __wrtcmp_dlon__="0.11691181" +## __wrtcmp_dlat__="0.11691181" +# __wrtcmp_dlon__=$( printf "%.9f" $( bc -l <<< "(${esggrid_delx}/${radius_Earth})*${degs_per_radian}" ) ) +# __wrtcmp_dlat__=$( printf "%.9f" $( bc -l <<< "(${esggrid_dely}/${radius_Earth})*${degs_per_radian}" ) ) +# fi + ;; # #----------------------------------------------------------------------- # @@ -546,92 +687,92 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_AK_3km") + "RRFS_AK_3km") -# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then +# if [ "${grid_gen_method}" = "GFDLgrid" ]; then # -# GFDLgrid_LON_T6_CTR="-160.8" -# GFDLgrid_LAT_T6_CTR="63.0" -# GFDLgrid_STRETCH_FAC="1.161" -# GFDLgrid_RES="768" -# GFDLgrid_REFINE_RATIO="4" +# __gfdlgrid_lon_t6_ctr__="-160.8" +# __gfdlgrid_lat_t6_ctr__="63.0" +# __gfdlgrid_stretch_fac__="1.161" +# __gfdlgrid_res__="768" +# __gfdlgrid_refine_ratio__="4" # -# num_margin_cells_T6_left="204" -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) +# num_margin_cells_T6_left="204" +# __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) # -# num_margin_cells_T6_right="204" -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) +# num_margin_cells_T6_right="204" +# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) # -# num_margin_cells_T6_bottom="249" -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) +# num_margin_cells_T6_bottom="249" +# __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) # -# num_margin_cells_T6_top="249" -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) +# num_margin_cells_T6_top="249" +# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" +# __gfdlgrid_use_gfdlgrid_res_in_filenames__="FALSE" # -# DT_ATMOS="${DT_ATMOS:-18}" +# __dt_atmos__="${dt_atmos:-18}" # -# LAYOUT_X="${LAYOUT_X:-24}" -# LAYOUT_Y="${LAYOUT_Y:-24}" -# BLOCKSIZE="${BLOCKSIZE:-15}" +# __layout_x__="${layout_x:-24}" +# __layout_y__="${layout_y:-24}" +# __blocksize__="${blocksize:-15}" # -# if [ "$QUILTING" = "TRUE" ]; then -# WRTCMP_write_groups="1" -# WRTCMP_write_tasks_per_group="2" -# WRTCMP_output_grid="lambert_conformal" -# WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}" -# WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}" -# WRTCMP_stdlat1="${GFDLgrid_LAT_T6_CTR}" -# WRTCMP_stdlat2="${GFDLgrid_LAT_T6_CTR}" -# WRTCMP_nx="1320" -# WRTCMP_ny="950" -# WRTCMP_lon_lwr_left="173.734" -# WRTCMP_lat_lwr_left="46.740347" -# WRTCMP_dx="3000.0" -# WRTCMP_dy="3000.0" -# fi +# if [ "$quilting" = "TRUE" ]; then +# __wrtcmp_write_groups__="1" +# __wrtcmp_write_tasks_per_group__="2" +# __wrtcmp_output_grid__="lambert_conformal" +# __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" +# __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" +# __wrtcmp_stdlat1__="${__gfdlgrid_lat_t6_ctr__}" +# __wrtcmp_stdlat2__="${__gfdlgrid_lat_t6_ctr__}" +# __wrtcmp_nx__="1320" +# __wrtcmp_ny__="950" +# __wrtcmp_lon_lwr_left__="173.734" +# __wrtcmp_lat_lwr_left__="46.740347" +# __wrtcmp_dx__="3000.0" +# __wrtcmp_dy__="3000.0" +# fi # -# elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then +# elif [ "${grid_gen_method}" = "ESGgrid" ]; then - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-161.5" - ESGgrid_LAT_CTR="63.0" + __esggrid_lon_ctr__="-161.5" + __esggrid_lat_ctr__="63.0" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="1380" - ESGgrid_NY="1020" + __esggrid_nx__="1380" + __esggrid_ny__="1020" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" -# DT_ATMOS="${DT_ATMOS:-50}" - DT_ATMOS="${DT_ATMOS:-10}" +# __dt_atmos__="${dt_atmos:-50}" + __dt_atmos__="${dt_atmos:-10}" - LAYOUT_X="${LAYOUT_X:-30}" - LAYOUT_Y="${LAYOUT_Y:-17}" - BLOCKSIZE="${BLOCKSIZE:-40}" + __layout_x__="${layout_x:-30}" + __layout_y__="${layout_y:-17}" + __blocksize__="${blocksize:-40}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="1379" - WRTCMP_ny="1003" - WRTCMP_lon_lwr_left="-187.89737923" - WRTCMP_lat_lwr_left="45.84576053" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="1379" + __wrtcmp_ny__="1003" + __wrtcmp_lon_lwr_left__="-187.89737923" + __wrtcmp_lat_lwr_left__="45.84576053" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -639,53 +780,53 @@ case ${PREDEF_GRID_NAME} in # # Note: # The WoFS domain will generate a 301 x 301 output grid (WRITE COMPONENT) and -# will eventually be movable (ESGgrid_LON_CTR/ESGgrid_LAT_CTR). A python script +# will eventually be movable (esggrid_lon_ctr/esggrid_lat_ctr). A python script # python_utils/fv3write_parms_lambert will be useful to determine -# WRTCMP_lon_lwr_left and WRTCMP_lat_lwr_left locations (only for Lambert map -# projection currently) of the quilting output when the domain location is +# wrtcmp_lon_lwr_left and wrtcmp_lat_lwr_left locations (only for Lambert map +# projection currently) of the quilting output when the domain location is # moved. Later, it should be integrated into the workflow. # #----------------------------------------------------------------------- # -"WoFS_3km") + "WoFS_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-97.5" - ESGgrid_LAT_CTR="38.5" + __esggrid_lon_ctr__="-97.5" + __esggrid_lat_ctr__="38.5" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="361" - ESGgrid_NY="361" + __esggrid_nx__="361" + __esggrid_ny__="361" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-20}" + __dt_atmos__="${dt_atmos:-20}" - LAYOUT_X="${LAYOUT_X:-18}" - LAYOUT_Y="${LAYOUT_Y:-12}" - BLOCKSIZE="${BLOCKSIZE:-30}" + __layout_x__="${layout_x:-18}" + __layout_y__="${layout_y:-12}" + __blocksize__="${blocksize:-30}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="301" - WRTCMP_ny="301" - WRTCMP_lon_lwr_left="-102.3802487" - WRTCMP_lat_lwr_left="34.3407918" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="301" + __wrtcmp_ny__="301" + __wrtcmp_lon_lwr_left__="-102.3802487" + __wrtcmp_lat_lwr_left__="34.3407918" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -697,50 +838,50 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"CONUS_25km_GFDLgrid") + "CONUS_25km_GFDLgrid") - GRID_GEN_METHOD="GFDLgrid" + __grid_gen_method__="GFDLgrid" - GFDLgrid_LON_T6_CTR="-97.5" - GFDLgrid_LAT_T6_CTR="38.5" - GFDLgrid_STRETCH_FAC="1.4" - GFDLgrid_RES="96" - GFDLgrid_REFINE_RATIO="3" + __gfdlgrid_lon_t6_ctr__="-97.5" + __gfdlgrid_lat_t6_ctr__="38.5" + __gfdlgrid_stretch_fac__="1.4" + __gfdlgrid_res__="96" + __gfdlgrid_refine_ratio__="3" - num_margin_cells_T6_left="12" - GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) + num_margin_cells_T6_left="12" + __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) - num_margin_cells_T6_right="12" - GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) + num_margin_cells_T6_right="12" + __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) - num_margin_cells_T6_bottom="16" - GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) + num_margin_cells_T6_bottom="16" + __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) - num_margin_cells_T6_top="16" - GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) + num_margin_cells_T6_top="16" + __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) - GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" + __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" - DT_ATMOS="${DT_ATMOS:-225}" + __dt_atmos__="${dt_atmos:-225}" - LAYOUT_X="${LAYOUT_X:-6}" - LAYOUT_Y="${LAYOUT_Y:-4}" - BLOCKSIZE="${BLOCKSIZE:-36}" + __layout_x__="${layout_x:-6}" + __layout_y__="${layout_y:-4}" + __blocksize__="${blocksize:-36}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}" - WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}" - WRTCMP_lon_lwr_left="-24.40085141" - WRTCMP_lat_lwr_left="-19.65624142" - WRTCMP_lon_upr_rght="24.40085141" - WRTCMP_lat_upr_rght="19.65624142" - WRTCMP_dlon="0.22593381" - WRTCMP_dlat="0.22593381" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="rotated_latlon" + __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" + __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" + __wrtcmp_lon_lwr_left__="-24.40085141" + __wrtcmp_lat_lwr_left__="-19.65624142" + __wrtcmp_lon_upr_rght__="24.40085141" + __wrtcmp_lat_upr_rght__="19.65624142" + __wrtcmp_dlon__="0.22593381" + __wrtcmp_dlat__="0.22593381" + fi + ;; # #----------------------------------------------------------------------- # @@ -752,50 +893,50 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"CONUS_3km_GFDLgrid") + "CONUS_3km_GFDLgrid") - GRID_GEN_METHOD="GFDLgrid" + __grid_gen_method__="GFDLgrid" - GFDLgrid_LON_T6_CTR="-97.5" - GFDLgrid_LAT_T6_CTR="38.5" - GFDLgrid_STRETCH_FAC="1.5" - GFDLgrid_RES="768" - GFDLgrid_REFINE_RATIO="3" + __gfdlgrid_lon_t6_ctr__="-97.5" + __gfdlgrid_lat_t6_ctr__="38.5" + __gfdlgrid_stretch_fac__="1.5" + __gfdlgrid_res__="768" + __gfdlgrid_refine_ratio__="3" - num_margin_cells_T6_left="69" - GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) + num_margin_cells_T6_left="69" + __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) - num_margin_cells_T6_right="69" - GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) + num_margin_cells_T6_right="69" + __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) - num_margin_cells_T6_bottom="164" - GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) + num_margin_cells_T6_bottom="164" + __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) - num_margin_cells_T6_top="164" - GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) + num_margin_cells_T6_top="164" + __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) - GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" + __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" - DT_ATMOS="${DT_ATMOS:-18}" + __dt_atmos__="${dt_atmos:-18}" - LAYOUT_X="${LAYOUT_X:-30}" - LAYOUT_Y="${LAYOUT_Y:-22}" - BLOCKSIZE="${BLOCKSIZE:-35}" + __layout_x__="${layout_x:-30}" + __layout_y__="${layout_y:-22}" + __blocksize__="${blocksize:-35}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) - WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="${GFDLgrid_LON_T6_CTR}" - WRTCMP_cen_lat="${GFDLgrid_LAT_T6_CTR}" - WRTCMP_lon_lwr_left="-25.23144805" - WRTCMP_lat_lwr_left="-15.82130419" - WRTCMP_lon_upr_rght="25.23144805" - WRTCMP_lat_upr_rght="15.82130419" - WRTCMP_dlon="0.02665763" - WRTCMP_dlat="0.02665763" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) + __wrtcmp_output_grid__="rotated_latlon" + __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" + __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" + __wrtcmp_lon_lwr_left__="-25.23144805" + __wrtcmp_lat_lwr_left__="-15.82130419" + __wrtcmp_lon_upr_rght__="25.23144805" + __wrtcmp_lat_upr_rght__="15.82130419" + __wrtcmp_dlon__="0.02665763" + __wrtcmp_dlat__="0.02665763" + fi + ;; # #----------------------------------------------------------------------- # @@ -803,9 +944,9 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"EMC_AK") + "EMC_AK") -# if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then +# if [ "${grid_gen_method}" = "GFDLgrid" ]; then # Values from an EMC script. @@ -841,61 +982,61 @@ case ${PREDEF_GRID_NAME} in #dlon=0.03 #dlat=0.03 -# GFDLgrid_LON_T6_CTR="-153.0" -# GFDLgrid_LAT_T6_CTR="61.0" -# GFDLgrid_STRETCH_FAC="1.0" # ??? -# GFDLgrid_RES="768" -# GFDLgrid_REFINE_RATIO="3" # ??? +# __gfdlgrid_lon_t6_ctr__="-153.0" +# __gfdlgrid_lat_t6_ctr__="61.0" +# __gfdlgrid_stretch_fac__="1.0" # ??? +# __gfdlgrid_res__="768" +# __gfdlgrid_refine_ratio__="3" # ??? # -# num_margin_cells_T6_left="61" -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) +# num_margin_cells_T6_left="61" +# __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) # -# num_margin_cells_T6_right="67" -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) +# num_margin_cells_T6_right="67" +# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) # -# num_margin_cells_T6_bottom="165" -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_bottom + 1 )) +# num_margin_cells_T6_bottom="165" +# __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) # -# num_margin_cells_T6_top="171" -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) +# num_margin_cells_T6_top="171" +# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" +# __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" # -# DT_ATMOS="${DT_ATMOS:-18}" +# __dt_atmos__="${dt_atmos:-18}" # -# LAYOUT_X="${LAYOUT_X:-16}" -# LAYOUT_Y="${LAYOUT_Y:-48}" -# WRTCMP_write_groups="2" -# WRTCMP_write_tasks_per_group="24" -# BLOCKSIZE="${BLOCKSIZE:-32}" +# __layout_x__="${layout_x:-16}" +# __layout_y__="${layout_y:-48}" +# __wrtcmp_write_groups__="2" +# __wrtcmp_write_tasks_per_group__="24" +# __blocksize__="${blocksize:-32}" # -# elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then +# elif [ "${grid_gen_method}" = "ESGgrid" ]; then - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR="-153.0" - ESGgrid_LAT_CTR="61.0" + __esggrid_lon_ctr__="-153.0" + __esggrid_lat_ctr__="61.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/ak/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX="1344" # Supergrid value 2704 - ESGgrid_NY="1152" # Supergrid value 2320 + __esggrid_nx__="1344" # Supergrid value 2704 + __esggrid_ny__="1152" # Supergrid value 2320 # Rotation of the ESG grid in degrees. - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -906,41 +1047,41 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="${DT_ATMOS:-18}" + __dt_atmos__="${dt_atmos:-18}" -#Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="${LAYOUT_X:-28}" - LAYOUT_Y="${LAYOUT_Y:-16}" +#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y + __layout_x__="${layout_x:-28}" + __layout_y__="${layout_y:-16}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="${BLOCKSIZE:-24}" + __blocksize__="${blocksize:-24}" #This section is all for the write component, which you need for output during model integration - if [ "$QUILTING" = "TRUE" ]; then + if [ "$quilting" = "TRUE" ]; then #Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - WRTCMP_write_groups="1" -#Number of tasks per write group. Ny must be divisible my this number. LAYOUT_Y is usually a good value - WRTCMP_write_tasks_per_group="24" + __wrtcmp_write_groups__="1" +#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value + __wrtcmp_write_tasks_per_group__="24" #lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - WRTCMP_output_grid="lambert_conformal" + __wrtcmp_output_grid__="lambert_conformal" #These should always be set the same as compute grid - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" #Write component grid must always be <= compute grid (without haloes) - WRTCMP_nx="1344" - WRTCMP_ny="1152" + __wrtcmp_nx__="1344" + __wrtcmp_ny__="1152" #Lower left latlon (southwest corner) - WRTCMP_lon_lwr_left="-177.0" - WRTCMP_lat_lwr_left="42.5" - WRTCMP_dx="$ESGgrid_DELX" - WRTCMP_dy="$ESGgrid_DELY" - fi - ;; + __wrtcmp_lon_lwr_left__="-177.0" + __wrtcmp_lat_lwr_left__="42.5" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -948,32 +1089,32 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"EMC_HI") + "EMC_HI") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/hi/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR="-157.0" - ESGgrid_LAT_CTR="20.0" + __esggrid_lon_ctr__="-157.0" + __esggrid_lat_ctr__="20.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/hi/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX="432" # Supergrid value 880 - ESGgrid_NY="360" # Supergrid value 736 + __esggrid_nx__="432" # Supergrid value 880 + __esggrid_ny__="360" # Supergrid value 736 # Rotation of the ESG grid in degrees. - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -984,41 +1125,41 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="${DT_ATMOS:-18}" + __dt_atmos__="${dt_atmos:-18}" -#Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="${LAYOUT_X:-8}" - LAYOUT_Y="${LAYOUT_Y:-8}" +#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y + __layout_x__="${layout_x:-8}" + __layout_y__="${layout_y:-8}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="${BLOCKSIZE:-27}" + __blocksize__="${blocksize:-27}" #This section is all for the write component, which you need for output during model integration - if [ "$QUILTING" = "TRUE" ]; then + if [ "$quilting" = "TRUE" ]; then #Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - WRTCMP_write_groups="1" -#Number of tasks per write group. Ny must be divisible my this number. LAYOUT_Y is usually a good value - WRTCMP_write_tasks_per_group="8" + __wrtcmp_write_groups__="1" +#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value + __wrtcmp_write_tasks_per_group__="8" #lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - WRTCMP_output_grid="lambert_conformal" + __wrtcmp_output_grid__="lambert_conformal" #These should usually be set the same as compute grid - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" #Write component grid should be close to the ESGgrid values unless you are doing something weird - WRTCMP_nx="420" - WRTCMP_ny="348" + __wrtcmp_nx__="420" + __wrtcmp_ny__="348" #Lower left latlon (southwest corner) - WRTCMP_lon_lwr_left="-162.8" - WRTCMP_lat_lwr_left="15.2" - WRTCMP_dx="$ESGgrid_DELX" - WRTCMP_dy="$ESGgrid_DELY" - fi - ;; + __wrtcmp_lon_lwr_left__="-162.8" + __wrtcmp_lat_lwr_left__="15.2" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -1026,32 +1167,32 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"EMC_PR") + "EMC_PR") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/pr/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR="-69.0" - ESGgrid_LAT_CTR="18.0" + __esggrid_lon_ctr__="-69.0" + __esggrid_lat_ctr__="18.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/pr/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX="576" # Supergrid value 1168 - ESGgrid_NY="432" # Supergrid value 880 + __esggrid_nx__="576" # Supergrid value 1168 + __esggrid_ny__="432" # Supergrid value 880 # Rotation of the ESG grid in degrees. - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -1062,41 +1203,41 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="${DT_ATMOS:-18}" + __dt_atmos__="${dt_atmos:-18}" -#Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-8}" +#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-8}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="${BLOCKSIZE:-24}" + __blocksize__="${blocksize:-24}" #This section is all for the write component, which you need for output during model integration - if [ "$QUILTING" = "TRUE" ]; then + if [ "$quilting" = "TRUE" ]; then #Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - WRTCMP_write_groups="1" -#Number of tasks per write group. Ny must be divisible my this number. LAYOUT_Y is usually a good value - WRTCMP_write_tasks_per_group="24" + __wrtcmp_write_groups__="1" +#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value + __wrtcmp_write_tasks_per_group__="24" #lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - WRTCMP_output_grid="lambert_conformal" + __wrtcmp_output_grid__="lambert_conformal" #These should always be set the same as compute grid - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" #Write component grid must always be <= compute grid (without haloes) - WRTCMP_nx="576" - WRTCMP_ny="432" + __wrtcmp_nx__="576" + __wrtcmp_ny__="432" #Lower left latlon (southwest corner) - WRTCMP_lon_lwr_left="-77" - WRTCMP_lat_lwr_left="12" - WRTCMP_dx="$ESGgrid_DELX" - WRTCMP_dy="$ESGgrid_DELY" - fi - ;; + __wrtcmp_lon_lwr_left__="-77" + __wrtcmp_lat_lwr_left__="12" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -1104,32 +1245,32 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"EMC_GU") + "EMC_GU") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" # Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/guam/C768_grid.tile7.nc # With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar # Longitude and latitude for center of domain - ESGgrid_LON_CTR="146.0" - ESGgrid_LAT_CTR="15.0" + __esggrid_lon_ctr__="146.0" + __esggrid_lat_ctr__="15.0" # Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution # of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" # Number of x and y points for your domain (halo not included); # Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/guam/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - ESGgrid_NX="432" # Supergrid value 880 - ESGgrid_NY="360" # Supergrid value 736 + __esggrid_nx__="432" # Supergrid value 880 + __esggrid_ny__="360" # Supergrid value 736 # Rotation of the ESG grid in degrees. - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" # Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now # Within the model we actually have a 4-point halo and a 3-point halo - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" # Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration # 'ksplit' is the factor that determines the timestep for this process (divided @@ -1140,40 +1281,40 @@ case ${PREDEF_GRID_NAME} in # # Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - DT_ATMOS="${DT_ATMOS:-18}" + __dt_atmos__="${dt_atmos:-18}" -#Factors for MPI decomposition. ESGgrid_NX must be divisible by LAYOUT_X, ESGgrid_NY must be divisible by LAYOUT_Y - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-12}" +#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-12}" #Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. #This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic # For Theia, must be ~40 or less # Check setup.sh for more details - BLOCKSIZE="${BLOCKSIZE:-27}" + __blocksize__="${blocksize:-27}" #This section is all for the write component, which you need for output during model integration - if [ "$QUILTING" = "TRUE" ]; then + if [ "$quilting" = "TRUE" ]; then #Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - WRTCMP_write_groups="1" -#Number of tasks per write group. Ny must be divisible my this number. LAYOUT_Y is usually a good value - WRTCMP_write_tasks_per_group="24" + __wrtcmp_write_groups__="1" +#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value + __wrtcmp_write_tasks_per_group__="24" #lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - WRTCMP_output_grid="lambert_conformal" + __wrtcmp_output_grid__="lambert_conformal" #These should always be set the same as compute grid - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" #Write component grid must always be <= compute grid (without haloes) - WRTCMP_nx="420" - WRTCMP_ny="348" + __wrtcmp_nx__="420" + __wrtcmp_ny__="348" #Lower left latlon (southwest corner) Used /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/dbrowse/fv3grid utility to find best value - WRTCMP_lon_lwr_left="140" - WRTCMP_lat_lwr_left="10" - WRTCMP_dx="$ESGgrid_DELX" - WRTCMP_dy="$ESGgrid_DELY" - fi - ;; + __wrtcmp_lon_lwr_left__="140" + __wrtcmp_lat_lwr_left__="10" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -1181,43 +1322,43 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"GSL_HAFSV0.A_25km") + "GSL_HAFSV0.A_25km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-62.0" - ESGgrid_LAT_CTR="22.0" + __esggrid_lon_ctr__="-62.0" + __esggrid_lat_ctr__="22.0" - ESGgrid_DELX="25000.0" - ESGgrid_DELY="25000.0" + __esggrid_delx__="25000.0" + __esggrid_dely__="25000.0" - ESGgrid_NX="345" - ESGgrid_NY="230" + __esggrid_nx__="345" + __esggrid_ny__="230" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-300}" + __dt_atmos__="${dt_atmos:-300}" - LAYOUT_X="${LAYOUT_X:-5}" - LAYOUT_Y="${LAYOUT_Y:-5}" - BLOCKSIZE="${BLOCKSIZE:-6}" + __layout_x__="${layout_x:-5}" + __layout_y__="${layout_y:-5}" + __blocksize__="${blocksize:-6}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="32" - WRTCMP_output_grid="regional_latlon" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="25.0" - WRTCMP_lon_lwr_left="-114.5" - WRTCMP_lat_lwr_left="-5.0" - WRTCMP_lon_upr_rght="-9.5" - WRTCMP_lat_upr_rght="55.0" - WRTCMP_dlon="0.25" - WRTCMP_dlat="0.25" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="32" + __wrtcmp_output_grid__="regional_latlon" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="25.0" + __wrtcmp_lon_lwr_left__="-114.5" + __wrtcmp_lat_lwr_left__="-5.0" + __wrtcmp_lon_upr_rght__="-9.5" + __wrtcmp_lat_upr_rght__="55.0" + __wrtcmp_dlon__="0.25" + __wrtcmp_dlat__="0.25" + fi + ;; # #----------------------------------------------------------------------- # @@ -1225,43 +1366,43 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"GSL_HAFSV0.A_13km") + "GSL_HAFSV0.A_13km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-62.0" - ESGgrid_LAT_CTR="22.0" + __esggrid_lon_ctr__="-62.0" + __esggrid_lat_ctr__="22.0" - ESGgrid_DELX="13000.0" - ESGgrid_DELY="13000.0" + __esggrid_delx__="13000.0" + __esggrid_dely__="13000.0" - ESGgrid_NX="665" - ESGgrid_NY="444" + __esggrid_nx__="665" + __esggrid_ny__="444" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-180}" + __dt_atmos__="${dt_atmos:-180}" - LAYOUT_X="${LAYOUT_X:-19}" - LAYOUT_Y="${LAYOUT_Y:-12}" - BLOCKSIZE="${BLOCKSIZE:-35}" + __layout_x__="${layout_x:-19}" + __layout_y__="${layout_y:-12}" + __blocksize__="${blocksize:-35}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="32" - WRTCMP_output_grid="regional_latlon" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="25.0" - WRTCMP_lon_lwr_left="-114.5" - WRTCMP_lat_lwr_left="-5.0" - WRTCMP_lon_upr_rght="-9.5" - WRTCMP_lat_upr_rght="55.0" - WRTCMP_dlon="0.13" - WRTCMP_dlat="0.13" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="32" + __wrtcmp_output_grid__="regional_latlon" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="25.0" + __wrtcmp_lon_lwr_left__="-114.5" + __wrtcmp_lat_lwr_left__="-5.0" + __wrtcmp_lon_upr_rght__="-9.5" + __wrtcmp_lat_upr_rght__="55.0" + __wrtcmp_dlon__="0.13" + __wrtcmp_dlat__="0.13" + fi + ;; # #----------------------------------------------------------------------- # @@ -1269,43 +1410,43 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"GSL_HAFSV0.A_3km") + "GSL_HAFSV0.A_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-62.0" - ESGgrid_LAT_CTR="22.0" + __esggrid_lon_ctr__="-62.0" + __esggrid_lat_ctr__="22.0" - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX="2880" - ESGgrid_NY="1920" + __esggrid_nx__="2880" + __esggrid_ny__="1920" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-40}" + __dt_atmos__="${dt_atmos:-40}" - LAYOUT_X="${LAYOUT_X:-32}" - LAYOUT_Y="${LAYOUT_Y:-24}" - BLOCKSIZE="${BLOCKSIZE:-32}" + __layout_x__="${layout_x:-32}" + __layout_y__="${layout_y:-24}" + __blocksize__="${blocksize:-32}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="32" - WRTCMP_output_grid="regional_latlon" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="25.0" - WRTCMP_lon_lwr_left="-114.5" - WRTCMP_lat_lwr_left="-5.0" - WRTCMP_lon_upr_rght="-9.5" - WRTCMP_lat_upr_rght="55.0" - WRTCMP_dlon="0.03" - WRTCMP_dlat="0.03" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="32" + __wrtcmp_output_grid__="regional_latlon" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="25.0" + __wrtcmp_lon_lwr_left__="-114.5" + __wrtcmp_lat_lwr_left__="-5.0" + __wrtcmp_lon_upr_rght__="-9.5" + __wrtcmp_lat_upr_rght__="55.0" + __wrtcmp_dlon__="0.03" + __wrtcmp_dlat__="0.03" + fi + ;; # #----------------------------------------------------------------------- # @@ -1313,45 +1454,45 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"GSD_HRRR_AK_50km") + "GSD_HRRR_AK_50km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-163.5" - ESGgrid_LAT_CTR="62.8" + __esggrid_lon_ctr__="-163.5" + __esggrid_lat_ctr__="62.8" - ESGgrid_DELX="50000.0" - ESGgrid_DELY="50000.0" + __esggrid_delx__="50000.0" + __esggrid_dely__="50000.0" - ESGgrid_NX="74" - ESGgrid_NY="51" + __esggrid_nx__="74" + __esggrid_ny__="51" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-600}" + __dt_atmos__="${dt_atmos:-600}" - LAYOUT_X="${LAYOUT_X:-2}" - LAYOUT_Y="${LAYOUT_Y:-3}" - BLOCKSIZE="${BLOCKSIZE:-37}" + __layout_x__="${layout_x:-2}" + __layout_y__="${layout_y:-3}" + __blocksize__="${blocksize:-37}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="1" - WRTCMP_output_grid="lambert_conformal" - WRTCMP_cen_lon="${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat1="${ESGgrid_LAT_CTR}" - WRTCMP_stdlat2="${ESGgrid_LAT_CTR}" - WRTCMP_nx="70" - WRTCMP_ny="45" - WRTCMP_lon_lwr_left="172.0" - WRTCMP_lat_lwr_left="49.0" - WRTCMP_dx="${ESGgrid_DELX}" - WRTCMP_dy="${ESGgrid_DELY}" - fi - ;; + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="1" + __wrtcmp_output_grid__="lambert_conformal" + __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" + __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" + __wrtcmp_nx__="70" + __wrtcmp_ny__="45" + __wrtcmp_lon_lwr_left__="172.0" + __wrtcmp_lat_lwr_left__="49.0" + __wrtcmp_dx__="${__esggrid_delx__}" + __wrtcmp_dy__="${__esggrid_dely__}" + fi + ;; # #----------------------------------------------------------------------- # @@ -1359,44 +1500,43 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_NA_13km") + "RRFS_NA_13km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR="-112.5" - ESGgrid_LAT_CTR="55.0" + __esggrid_lon_ctr__="-112.5" + __esggrid_lat_ctr__="55.0" - ESGgrid_DELX="13000.0" - ESGgrid_DELY="13000.0" + __esggrid_delx__="13000.0" + __esggrid_dely__="13000.0" - ESGgrid_NX="912" - ESGgrid_NY="623" + __esggrid_nx__="912" + __esggrid_ny__="623" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH="6" + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-50}" + __dt_atmos__="${dt_atmos:-50}" - LAYOUT_X="${LAYOUT_X:-16}" - LAYOUT_Y="${LAYOUT_Y:-16}" - BLOCKSIZE="${BLOCKSIZE:-30}" - - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="16" - WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-61.0" - WRTCMP_lat_lwr_left="-37.0" - WRTCMP_lon_upr_rght="61.0" - WRTCMP_lat_upr_rght="37.0" - WRTCMP_dlon=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) - WRTCMP_dlat=$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) - fi - ;; + __layout_x__="${layout_x:-16}" + __layout_y__="${layout_y:-16}" + __blocksize__="${blocksize:-30}" + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="16" + __wrtcmp_output_grid__="rotated_latlon" + __wrtcmp_cen_lon__="-113.0" #"${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="55.0" #"${__esggrid_lat_ctr__}" + __wrtcmp_lon_lwr_left__="-61.0" + __wrtcmp_lat_lwr_left__="-37.0" + __wrtcmp_lon_upr_rght__="61.0" + __wrtcmp_lat_upr_rght__="37.0" + __wrtcmp_dlon__=$( printf "%.9f" $( bc -l <<< "(${__esggrid_delx__}/${radius_Earth})*${degs_per_radian}" ) ) + __wrtcmp_dlat__=$( printf "%.9f" $( bc -l <<< "(${__esggrid_dely__}/${radius_Earth})*${degs_per_radian}" ) ) + fi + ;; # #----------------------------------------------------------------------- # @@ -1404,61 +1544,222 @@ case ${PREDEF_GRID_NAME} in # #----------------------------------------------------------------------- # -"RRFS_NA_3km") + "RRFS_NA_3km") - GRID_GEN_METHOD="ESGgrid" + __grid_gen_method__="ESGgrid" - ESGgrid_LON_CTR=-112.5 - ESGgrid_LAT_CTR=55.0 + __esggrid_lon_ctr__=-112.5 + __esggrid_lat_ctr__=55.0 - ESGgrid_DELX="3000.0" - ESGgrid_DELY="3000.0" + __esggrid_delx__="3000.0" + __esggrid_dely__="3000.0" - ESGgrid_NX=3950 - ESGgrid_NY=2700 + __esggrid_nx__="3950" + __esggrid_ny__="2700" - ESGgrid_PAZI="0.0" + __esggrid_pazi__="0.0" - ESGgrid_WIDE_HALO_WIDTH=6 + __esggrid_wide_halo_width__="6" - DT_ATMOS="${DT_ATMOS:-36}" + __dt_atmos__="${dt_atmos:-36}" - LAYOUT_X="${LAYOUT_X:-20}" # 40 - EMC operational configuration - LAYOUT_Y="${LAYOUT_Y:-35}" # 45 - EMC operational configuration - BLOCKSIZE="${BLOCKSIZE:-28}" + __layout_x__="${layout_x:-20}" # 40 - EMC operational configuration + __layout_y__="${layout_y:-35}" # 45 - EMC operational configuration + __blocksize__="${blocksize:-28}" - if [ "$QUILTING" = "TRUE" ]; then - WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group="144" - WRTCMP_output_grid="rotated_latlon" - WRTCMP_cen_lon="-113.0" #"${ESGgrid_LON_CTR}" - WRTCMP_cen_lat="55.0" #"${ESGgrid_LAT_CTR}" - WRTCMP_lon_lwr_left="-61.0" - WRTCMP_lat_lwr_left="-37.0" - WRTCMP_lon_upr_rght="61.0" - WRTCMP_lat_upr_rght="37.0" - WRTCMP_dlon="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELX}/${radius_Earth})*${degs_per_radian}" ) ) - WRTCMP_dlat="0.025" #$( printf "%.9f" $( bc -l <<< "(${ESGgrid_DELY}/${radius_Earth})*${degs_per_radian}" ) ) - fi - ;; -esac + if [ "$quilting" = "TRUE" ]; then + __wrtcmp_write_groups__="1" + __wrtcmp_write_tasks_per_group__="144" + __wrtcmp_output_grid__="rotated_latlon" + __wrtcmp_cen_lon__="-113.0" #"${__esggrid_lon_ctr__}" + __wrtcmp_cen_lat__="55.0" #"${__esggrid_lat_ctr__}" + __wrtcmp_lon_lwr_left__="-61.0" + __wrtcmp_lat_lwr_left__="-37.0" + __wrtcmp_lon_upr_rght__="61.0" + __wrtcmp_lat_upr_rght__="37.0" + __wrtcmp_dlon__="0.025" #$( printf "%.9f" $( bc -l <<< "(${__esggrid_delx__}/${radius_Earth})*${degs_per_radian}" ) ) + __wrtcmp_dlat__="0.025" #$( printf "%.9f" $( bc -l <<< "(${__esggrid_dely__}/${radius_Earth})*${degs_per_radian}" ) ) + fi + ;; + + esac # #----------------------------------------------------------------------- # -# Restore the shell options saved at the beginning of this script/func- -# tion. +# Use the printf utility with the -v flag to set this function's output +# variables. Note that each of these is set only if the corresponding +# input variable specifying the name to use for the output variable is +# not empty. # #----------------------------------------------------------------------- # -{ restore_shell_opts; } > /dev/null 2>&1 + if [ ! -z "${outvarname_grid_gen_method}" ]; then + printf -v ${outvarname_grid_gen_method} "%s" "${__grid_gen_method__}" + fi -} + if [ ! -z "${outvarname_esggrid_lon_ctr}" ]; then + printf -v ${outvarname_esggrid_lon_ctr} "%s" "${__esggrid_lon_ctr__}" + fi + + if [ ! -z "${outvarname_esggrid_lat_ctr}" ]; then + printf -v ${outvarname_esggrid_lat_ctr} "%s" "${__esggrid_lat_ctr__}" + fi + + if [ ! -z "${outvarname_esggrid_delx}" ]; then + printf -v ${outvarname_esggrid_delx} "%s" "${__esggrid_delx__}" + fi + + if [ ! -z "${outvarname_esggrid_dely}" ]; then + printf -v ${outvarname_esggrid_dely} "%s" "${__esggrid_dely__}" + fi + + if [ ! -z "${outvarname_esggrid_nx}" ]; then + printf -v ${outvarname_esggrid_nx} "%s" "${__esggrid_nx__}" + fi + + if [ ! -z "${outvarname_esggrid_ny}" ]; then + printf -v ${outvarname_esggrid_ny} "%s" "${__esggrid_ny__}" + fi + + if [ ! -z "${outvarname_esggrid_pazi}" ]; then + printf -v ${outvarname_esggrid_pazi} "%s" "${__esggrid_pazi__}" + fi + + if [ ! -z "${outvarname_esggrid_wide_halo_width}" ]; then + printf -v ${outvarname_esggrid_wide_halo_width} "%s" "${__esggrid_wide_halo_width__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_lon_t6_ctr}" ]; then + printf -v ${outvarname_gfdlgrid_lon_t6_ctr} "%s" "${__gfdlgrid_lon_t6_ctr__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_lat_t6_ctr}" ]; then + printf -v ${outvarname_gfdlgrid_lat_t6_ctr} "%s" "${__gfdlgrid_lat_t6_ctr__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_stretch_fac}" ]; then + printf -v ${outvarname_gfdlgrid_stretch_fac} "%s" "${__gfdlgrid_stretch_fac__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_res}" ]; then + printf -v ${outvarname_gfdlgrid_res} "%s" "${__gfdlgrid_res__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_refine_ratio}" ]; then + printf -v ${outvarname_gfdlgrid_refine_ratio} "%s" "${__gfdlgrid_refine_ratio__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g}" ]; then + printf -v ${outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_istart_of_rgnl_dom_on_t6g__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g}" ]; then + printf -v ${outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_iend_of_rgnl_dom_on_t6g__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g}" ]; then + printf -v ${outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_jstart_of_rgnl_dom_on_t6g__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g}" ]; then + printf -v ${outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_jend_of_rgnl_dom_on_t6g__}" + fi + + if [ ! -z "${outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames}" ]; then + printf -v ${outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames} "%s" "${__gfdlgrid_use_gfdlgrid_res_in_filenames__}" + fi + + if [ ! -z "${outvarname_dt_atmos}" ]; then + printf -v ${outvarname_dt_atmos} "%s" "${__dt_atmos__}" + fi + + if [ ! -z "${outvarname_layout_x}" ]; then + printf -v ${outvarname_layout_x} "%s" "${__layout_x__}" + fi + + if [ ! -z "${outvarname_layout_y}" ]; then + printf -v ${outvarname_layout_y} "%s" "${__layout_y__}" + fi + + if [ ! -z "${outvarname_blocksize}" ]; then + printf -v ${outvarname_blocksize} "%s" "${__blocksize__}" + fi + + if [ ! -z "${outvarname_wrtcmp_write_groups}" ]; then + printf -v ${outvarname_wrtcmp_write_groups} "%s" "${__wrtcmp_write_groups__}" + fi + + if [ ! -z "${outvarname_wrtcmp_write_tasks_per_group}" ]; then + printf -v ${outvarname_wrtcmp_write_tasks_per_group} "%s" "${__wrtcmp_write_tasks_per_group__}" + fi + + if [ ! -z "${outvarname_wrtcmp_output_grid}" ]; then + printf -v ${outvarname_wrtcmp_output_grid} "%s" "${__wrtcmp_output_grid__}" + fi + + if [ ! -z "${outvarname_wrtcmp_cen_lon}" ]; then + printf -v ${outvarname_wrtcmp_cen_lon} "%s" "${__wrtcmp_cen_lon__}" + fi + + if [ ! -z "${outvarname_wrtcmp_cen_lat}" ]; then + printf -v ${outvarname_wrtcmp_cen_lat} "%s" "${__wrtcmp_cen_lat__}" + fi + + if [ ! -z "${outvarname_wrtcmp_stdlat1}" ]; then + printf -v ${outvarname_wrtcmp_stdlat1} "%s" "${__wrtcmp_stdlat1__}" + fi + + if [ ! -z "${outvarname_wrtcmp_stdlat2}" ]; then + printf -v ${outvarname_wrtcmp_stdlat2} "%s" "${__wrtcmp_stdlat2__}" + fi + + if [ ! -z "${outvarname_wrtcmp_nx}" ]; then + printf -v ${outvarname_wrtcmp_nx} "%s" "${__wrtcmp_nx__}" + fi + + if [ ! -z "${outvarname_wrtcmp_ny}" ]; then + printf -v ${outvarname_wrtcmp_ny} "%s" "${__wrtcmp_ny__}" + fi + + if [ ! -z "${outvarname_wrtcmp_lon_lwr_left}" ]; then + printf -v ${outvarname_wrtcmp_lon_lwr_left} "%s" "${__wrtcmp_lon_lwr_left__}" + fi + + if [ ! -z "${outvarname_wrtcmp_lat_lwr_left}" ]; then + printf -v ${outvarname_wrtcmp_lat_lwr_left} "%s" "${__wrtcmp_lat_lwr_left__}" + fi + + if [ ! -z "${outvarname_wrtcmp_lon_upr_rght}" ]; then + printf -v ${outvarname_wrtcmp_lon_upr_rght} "%s" "${__wrtcmp_lon_upr_rght__}" + fi + + if [ ! -z "${outvarname_wrtcmp_lat_upr_rght}" ]; then + printf -v ${outvarname_wrtcmp_lat_upr_rght} "%s" "${__wrtcmp_lat_upr_rght__}" + fi + + if [ ! -z "${outvarname_wrtcmp_dx}" ]; then + printf -v ${outvarname_wrtcmp_dx} "%s" "${__wrtcmp_dx__}" + fi + + if [ ! -z "${outvarname_wrtcmp_dy}" ]; then + printf -v ${outvarname_wrtcmp_dy} "%s" "${__wrtcmp_dy__}" + fi + + if [ ! -z "${outvarname_wrtcmp_dlon}" ]; then + printf -v ${outvarname_wrtcmp_dlon} "%s" "${__wrtcmp_dlon__}" + fi + + if [ ! -z "${outvarname_wrtcmp_dlat}" ]; then + printf -v ${outvarname_wrtcmp_dlat} "%s" "${__wrtcmp_dlat__}" + fi # #----------------------------------------------------------------------- # -# Call the function defined above. +# Restore the shell options saved at the beginning of this script/func- +# tion. # #----------------------------------------------------------------------- # -set_predef_grid_params + { restore_shell_opts; } > /dev/null 2>&1 +} diff --git a/ush/setup.sh b/ush/setup.sh index a5e0f15fda..e15dad5c28 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -66,6 +66,7 @@ Starting function ${func_name}() in \"${scrfunc_fn}\"... # . ./check_expt_config_vars.sh . ./set_cycle_dates.sh +. ./set_predef_grid_params.sh . ./set_gridparams_GFDLgrid.sh . ./set_gridparams_ESGgrid.sh . ./link_fix.sh @@ -136,17 +137,20 @@ fi # Make sure that user-defined variables are set to valid values # # Set binary switch variables to either "TRUE" or "FALSE" by calling -# boolify so we don't have to consider other valid values later on +# boolify so we don't have to consider other valid values later on. # #----------------------------------------------------------------------- # check_var_valid_value "RUN_ENVIR" "valid_vals_RUN_ENVIR" -check_var_valid_value "VERBOSE" "valid_vals_VERBOSE" -VERBOSE=$(boolify $VERBOSE) +check_var_valid_value "VERBOSE" "valid_vals_BOOLEAN" +VERBOSE=$(boolify "$VERBOSE") -check_var_valid_value "DEBUG" "valid_vals_DEBUG" -DEBUG=$(boolify $DEBUG) +check_var_valid_value "DEBUG" "valid_vals_BOOLEAN" +DEBUG=$(boolify "$DEBUG") + +check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_BOOLEAN" +USE_CRON_TO_RELAUNCH=$(boolify "${USE_CRON_TO_RELAUNCH}") # #----------------------------------------------------------------------- # @@ -160,53 +164,78 @@ if [ "$DEBUG" = "TRUE" ]; then Setting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." VERBOSE="TRUE" fi +# +#----------------------------------------------------------------------- +# +# Check flags that turn on/off various workflow tasks. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_BOOLEAN" +RUN_TASK_MAKE_GRID=$(boolify "${RUN_TASK_MAKE_GRID}") -check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_USE_CRON_TO_RELAUNCH" -USE_CRON_TO_RELAUNCH=$(boolify $USE_CRON_TO_RELAUNCH) +check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_BOOLEAN" +RUN_TASK_MAKE_OROG=$(boolify "${RUN_TASK_MAKE_OROG}") -check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_RUN_TASK_MAKE_GRID" -RUN_TASK_MAKE_GRID=$(boolify $RUN_TASK_MAKE_GRID) +check_var_valid_value "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_BOOLEAN" +RUN_TASK_MAKE_SFC_CLIMO=$(boolify "${RUN_TASK_MAKE_SFC_CLIMO}") -check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_RUN_TASK_MAKE_OROG" -RUN_TASK_MAKE_OROG=$(boolify $RUN_TASK_MAKE_OROG) +check_var_valid_value "RUN_TASK_GET_EXTRN_ICS" "valid_vals_BOOLEAN" +RUN_TASK_GET_EXTRN_ICS=$(boolify "${RUN_TASK_GET_EXTRN_ICS}") -check_var_valid_value \ - "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_RUN_TASK_MAKE_SFC_CLIMO" -RUN_TASK_MAKE_SFC_CLIMO=$(boolify $RUN_TASK_MAKE_SFC_CLIMO) +check_var_valid_value "RUN_TASK_GET_EXTRN_LBCS" "valid_vals_BOOLEAN" +RUN_TASK_GET_EXTRN_LBCS=$(boolify "${RUN_TASK_GET_EXTRN_LBCS}") -check_var_valid_value \ - "RUN_TASK_RUN_POST" "valid_vals_RUN_TASK_RUN_POST" -RUN_TASK_RUN_POST=$(boolify $RUN_TASK_RUN_POST) +check_var_valid_value "RUN_TASK_RUN_FCST" "valid_vals_BOOLEAN" +RUN_TASK_RUN_FCST=$(boolify "${RUN_TASK_RUN_FCST}") + +check_var_valid_value "RUN_TASK_RUN_POST" "valid_vals_BOOLEAN" +RUN_TASK_RUN_POST=$(boolify "${RUN_TASK_RUN_POST}") -check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_RUN_TASK_VX_GRIDSTAT" -RUN_TASK_VX_GRIDSTAT=$(boolify $RUN_TASK_VX_GRIDSTAT) +check_var_valid_value "RUN_TASK_GET_OBS_CCPA" "valid_vals_BOOLEAN" +RUN_TASK_GET_OBS_CCPA=$(boolify "${RUN_TASK_GET_OBS_CCPA}") -check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_RUN_TASK_VX_POINTSTAT" -RUN_TASK_VX_POINTSTAT=$(boolify $RUN_TASK_VX_POINTSTAT) +check_var_valid_value "RUN_TASK_GET_OBS_MRMS" "valid_vals_BOOLEAN" +RUN_TASK_GET_OBS_MRMS=$(boolify "${RUN_TASK_GET_OBS_MRMS}") -check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_RUN_TASK_VX_ENSGRID" -RUN_TASK_VX_ENSGRID=$(boolify $RUN_TASK_VX_ENSGRID) +check_var_valid_value "RUN_TASK_GET_OBS_NDAS" "valid_vals_BOOLEAN" +RUN_TASK_GET_OBS_NDAS=$(boolify "${RUN_TASK_GET_OBS_NDAS}") -check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_RUN_TASK_VX_ENSPOINT" -RUN_TASK_VX_ENSPOINT=$(boolify $RUN_TASK_VX_ENSPOINT) +check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_BOOLEAN" +RUN_TASK_VX_GRIDSTAT=$(boolify "${RUN_TASK_VX_GRIDSTAT}") -check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" -USE_FVCOM=$(boolify $USE_FVCOM) +check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_BOOLEAN" +RUN_TASK_VX_POINTSTAT=$(boolify "${RUN_TASK_VX_POINTSTAT}") -check_var_valid_value "DO_SHUM" "valid_vals_DO_SHUM" -DO_SHUM=$(boolify $DO_SHUM) +check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_BOOLEAN" +RUN_TASK_VX_ENSGRID=$(boolify "${RUN_TASK_VX_ENSGRID}") -check_var_valid_value "DO_SPPT" "valid_vals_DO_SPPT" -DO_SPPT=$(boolify $DO_SPPT) +check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_BOOLEAN" +RUN_TASK_VX_ENSPOINT=$(boolify "${RUN_TASK_VX_ENSPOINT}") +# +#----------------------------------------------------------------------- +# +# Check stochastic physcs flags. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DO_SHUM" "valid_vals_BOOLEAN" +DO_SHUM=$(boolify "${DO_SHUM}") + +check_var_valid_value "DO_SPPT" "valid_vals_BOOLEAN" +DO_SPPT=$(boolify "${DO_SPPT}") + +check_var_valid_value "DO_SKEB" "valid_vals_BOOLEAN" +DO_SKEB=$(boolify "${DO_SKEB}") -check_var_valid_value "DO_SKEB" "valid_vals_DO_SKEB" -DO_SKEB=$(boolify $DO_SKEB) +check_var_valid_value "DO_SPP" "valid_vals_BOOLEAN" +DO_SPP=$(boolify "${DO_SPP}") -check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" -DO_SPP=$(boolify $DO_SPP) +check_var_valid_value "DO_LSM_SPP" "valid_vals_BOOLEAN" +DO_LSM_SPP=$(boolify "${DO_LSM_SPP}") -check_var_valid_value "DO_LSM_SPP" "valid_vals_DO_LSM_SPP" -DO_LSM_SPP=$(boolify $DO_LSM_SPP) +check_var_valid_value "USE_ZMTNBLCK" "valid_vals_BOOLEAN" +USE_ZMTNBLCK=$(boolify "${USE_ZMTNBLCK}") # #----------------------------------------------------------------------- # @@ -310,11 +339,6 @@ fi # #----------------------------------------------------------------------- # -check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" -SUB_HOURLY_POST=$(boolify $SUB_HOURLY_POST) -# -#----------------------------------------------------------------------- -# # Make sure that DOT_OR_USCORE is set to a valid value. # #----------------------------------------------------------------------- @@ -330,8 +354,9 @@ check_var_valid_value "DOT_OR_USCORE" "valid_vals_DOT_OR_USCORE" # #----------------------------------------------------------------------- # -check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" -USE_FVCOM=$(boolify $USE_FVCOM) +check_var_valid_value "USE_FVCOM" "valid_vals_BOOLEAN" +USE_FVCOM=$(boolify "${USE_FVCOM}") + check_var_valid_value "FVCOM_WCSTART" "valid_vals_FVCOM_WCSTART" FVCOM_WCSTART=$(echo_lowercase $FVCOM_WCSTART) # @@ -461,7 +486,6 @@ The base directory in which the UPP source code should be located Please clone the external repository containing the code in this directory, build the executable, and then rerun the workflow." fi - # # Define some other useful paths # @@ -477,7 +501,6 @@ TEMPLATE_DIR="$USHDIR/templates" VX_CONFIG_DIR="$TEMPLATE_DIR/parm" METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" MET_CONFIG="$TEMPLATE_DIR/parm/met" - # #----------------------------------------------------------------------- # @@ -518,15 +541,22 @@ One or more fix file directories have not been specified for this machine: DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR:-\"\"} You can specify the missing location(s) in ${machine_file}" fi - - # #----------------------------------------------------------------------- # -# Set the names of the build and workflow module files (if not -# already specified by the user). These are the files that need to be -# sourced before building the component SRW App codes and running various -# workflow scripts, respectively. +# Make sure COMPILER is set to a valid value. +# +#----------------------------------------------------------------------- +# +COMPILER=$(echo_lowercase $COMPILER) +check_var_valid_value "COMPILER" "valid_vals_COMPILER" +# +#----------------------------------------------------------------------- +# +# Set the names of the build and workflow module files (if not already +# specified by the user). These are the files that need to be loaded +# before building the component SRW App codes and running various workflow +# scripts, respectively. # #----------------------------------------------------------------------- # @@ -547,7 +577,7 @@ PPN_RUN_FCST=${PPN_RUN_FCST:-${ppn_run_fcst_default}} # #----------------------------------------------------------------------- # -# Make sure that the job scheduler set above is valid. +# Make sure SCHED is set to a valid value. # #----------------------------------------------------------------------- # @@ -634,12 +664,8 @@ check_var_valid_value \ # #----------------------------------------------------------------------- # -check_var_valid_value "USE_MERRA_CLIMO" "valid_vals_USE_MERRA_CLIMO" -# -# Set USE_MERRA_CLIMO to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -USE_MERRA_CLIMO=$(boolify $USE_MERRA_CLIMO) +check_var_valid_value "USE_MERRA_CLIMO" "valid_vals_BOOLEAN" +USE_MERRA_CLIMO=$(boolify "${USE_MERRA_CLIMO}") # Force to "TRUE" in case of FV3_GFS_v15_thompson_mynn_lam3km: if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15_thompson_mynn_lam3km" ]; then USE_MERRA_CLIMO="TRUE" @@ -814,6 +840,9 @@ fi # #----------------------------------------------------------------------- # +check_var_valid_value "USE_CUSTOM_POST_CONFIG_FILE" "valid_vals_BOOLEAN" +USE_CUSTOM_POST_CONFIG_FILE=$(boolify "${USE_CUSTOM_POST_CONFIG_FILE}") + if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then if [ ! -f "${CUSTOM_POST_CONFIG_FP}" ]; then print_err_msg_exit " @@ -825,12 +854,16 @@ fi # #----------------------------------------------------------------------- # -# If using external CRTM fix files to allow post-processing of synthetic -# satellite products from the UPP, then make sure the fix file directory -# exists. +# Ensure that USE_CRTM is set to a valid value. Then, if it is set to +# "TRUE" (i.e. if using external CRTM fix files to allow post-processing +# of synthetic satellite products from the UPP, make sure that the fix +# file directory exists. # #----------------------------------------------------------------------- # +check_var_valid_value "USE_CRTM" "valid_vals_BOOLEAN" +USE_CRTM=$(boolify "${USE_CRTM}") + if [ ${USE_CRTM} = "TRUE" ]; then if [ ! -d "${CRTM_DIR}" ]; then print_err_msg_exit " @@ -889,13 +922,62 @@ LBC_SPEC_FCST_HRS=($( seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} \ # #----------------------------------------------------------------------- # -# If PREDEF_GRID_NAME is set to a non-empty string, set or reset parameters -# according to the predefined domain specified. +# If PREDEF_GRID_NAME is set to a non-empty string, set or reset native +# and write-component grid parameters according to the specified predefined +# domain. # #----------------------------------------------------------------------- # if [ ! -z "${PREDEF_GRID_NAME}" ]; then - . $USHDIR/set_predef_grid_params.sh + + set_predef_grid_params \ + predef_grid_name="${PREDEF_GRID_NAME}" \ + dt_atmos="${DT_ATMOS}" \ + layout_x="${LAYOUT_X}" \ + layout_y="${LAYOUT_Y}" \ + blocksize="${BLOCKSIZE}" \ + quilting="${QUILTING}" \ + outvarname_grid_gen_method="GRID_GEN_METHOD" \ + outvarname_esggrid_lon_ctr="ESGgrid_LON_CTR" \ + outvarname_esggrid_lat_ctr="ESGgrid_LAT_CTR" \ + outvarname_esggrid_delx="ESGgrid_DELX" \ + outvarname_esggrid_dely="ESGgrid_DELY" \ + outvarname_esggrid_nx="ESGgrid_NX" \ + outvarname_esggrid_ny="ESGgrid_NY" \ + outvarname_esggrid_pazi="ESGgrid_PAZI" \ + outvarname_esggrid_wide_halo_width="ESGgrid_WIDE_HALO_WIDTH" \ + outvarname_gfdlgrid_lon_t6_ctr="GFDLgrid_LON_T6_CTR" \ + outvarname_gfdlgrid_lat_t6_ctr="GFDLgrid_LAT_T6_CTR" \ + outvarname_gfdlgrid_stretch_fac="GFDLgrid_STRETCH_FAC" \ + outvarname_gfdlgrid_res="GFDLgrid_RES" \ + outvarname_gfdlgrid_refine_ratio="GFDLgrid_REFINE_RATIO" \ + outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G" \ + outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G" \ + outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g="GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G" \ + outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g="GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G" \ + outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames="GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES" \ + outvarname_dt_atmos="DT_ATMOS" \ + outvarname_layout_x="LAYOUT_X" \ + outvarname_layout_y="LAYOUT_Y" \ + outvarname_blocksize="BLOCKSIZE" \ + outvarname_wrtcmp_write_groups="WRTCMP_write_groups" \ + outvarname_wrtcmp_write_tasks_per_group="WRTCMP_write_tasks_per_group" \ + outvarname_wrtcmp_output_grid="WRTCMP_output_grid" \ + outvarname_wrtcmp_cen_lon="WRTCMP_cen_lon" \ + outvarname_wrtcmp_cen_lat="WRTCMP_cen_lat" \ + outvarname_wrtcmp_stdlat1="WRTCMP_stdlat1" \ + outvarname_wrtcmp_stdlat2="WRTCMP_stdlat2" \ + outvarname_wrtcmp_nx="WRTCMP_nx" \ + outvarname_wrtcmp_ny="WRTCMP_ny" \ + outvarname_wrtcmp_lon_lwr_left="WRTCMP_lon_lwr_left" \ + outvarname_wrtcmp_lat_lwr_left="WRTCMP_lat_lwr_left" \ + outvarname_wrtcmp_lon_upr_rght="WRTCMP_lon_upr_rght" \ + outvarname_wrtcmp_lat_upr_rght="WRTCMP_lat_upr_rght" \ + outvarname_wrtcmp_dx="WRTCMP_dx" \ + outvarname_wrtcmp_dy="WRTCMP_dy" \ + outvarname_wrtcmp_dlon="WRTCMP_dlon" \ + outvarname_wrtcmp_dlat="WRTCMP_dlat" + fi # #----------------------------------------------------------------------- @@ -982,6 +1064,9 @@ fi # #----------------------------------------------------------------------- # +check_var_valid_value "SUB_HOURLY_POST" "valid_vals_BOOLEAN" +SUB_HOURLY_POST=$(boolify "${SUB_HOURLY_POST}") + if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then # # Check that DT_SUBHOURLY_POST_MNTS is a string consisting of one or two @@ -1351,11 +1436,6 @@ FIELD_TABLE_FP="${EXPTDIR}/${FIELD_TABLE_FN}" FV3_NML_FN="${FV3_NML_BASE_SUITE_FN%.*}" FV3_NML_FP="${EXPTDIR}/${FV3_NML_FN}" NEMS_CONFIG_FP="${EXPTDIR}/${NEMS_CONFIG_FN}" - - -check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_USE_USER_STAGED_EXTRN_FILES" -USE_USER_STAGED_EXTRN_FILES=$(boolify $USE_USER_STAGED_EXTRN_FILES) - # #----------------------------------------------------------------------- # @@ -1365,6 +1445,9 @@ USE_USER_STAGED_EXTRN_FILES=$(boolify $USE_USER_STAGED_EXTRN_FILES) # #----------------------------------------------------------------------- # +check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_BOOLEAN" +USE_USER_STAGED_EXTRN_FILES=$(boolify "${USE_USER_STAGED_EXTRN_FILES}") + if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then # Check for the base directory up to the first templated field. @@ -1383,6 +1466,9 @@ external model files for generating LBCs should be located does not exist: fi fi + +check_var_valid_value "NOMADS" "valid_vals_BOOLEAN" +NOMADS=$(boolify "${NOMADS}") # #----------------------------------------------------------------------- # @@ -1393,8 +1479,8 @@ fi # #----------------------------------------------------------------------- # -check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" -DO_ENSEMBLE=$(boolify $DO_ENSEMBLE) +check_var_valid_value "DO_ENSEMBLE" "valid_vals_BOOLEAN" +DO_ENSEMBLE=$(boolify "${DO_ENSEMBLE}") NDIGITS_ENSMEM_NAMES="0" ENSMEM_NAMES=("") @@ -1796,45 +1882,6 @@ fi # #----------------------------------------------------------------------- # -# Any regional model must be supplied lateral boundary conditions (in -# addition to initial conditions) to be able to perform a forecast. In -# the FV3-LAM model, these boundary conditions (BCs) are supplied using a -# "halo" of grid cells around the regional domain that extend beyond the -# boundary of the domain. The model is formulated such that along with -# files containing these BCs, it needs as input the following files (in -# NetCDF format): -# -# 1) A grid file that includes a halo of 3 cells beyond the boundary of -# the domain. -# 2) A grid file that includes a halo of 4 cells beyond the boundary of -# the domain. -# 3) A (filtered) orography file without a halo, i.e. a halo of width -# 0 cells. -# 4) A (filtered) orography file that includes a halo of 4 cells beyond -# the boundary of the domain. -# -# Note that the regional grid is referred to as "tile 7" in the code. -# We will let: -# -# * NH0 denote the width (in units of number of cells on tile 7) of -# the 0-cell-wide halo, i.e. NH0 = 0; -# -# * NH3 denote the width (in units of number of cells on tile 7) of -# the 3-cell-wide halo, i.e. NH3 = 3; and -# -# * NH4 denote the width (in units of number of cells on tile 7) of -# the 4-cell-wide halo, i.e. NH4 = 4. -# -# We define these variables next. -# -#----------------------------------------------------------------------- -# -NH0=0 -NH3=3 -NH4=4 -# -#----------------------------------------------------------------------- -# # Set parameters according to the type of horizontal grid generation # method specified. First consider GFDL's global-parent-grid based # method. @@ -1853,16 +1900,17 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then iend_of_t7_on_t6g="${GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G}" \ jstart_of_t7_on_t6g="${GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G}" \ jend_of_t7_on_t6g="${GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G}" \ - output_varname_lon_of_t7_ctr="LON_CTR" \ - output_varname_lat_of_t7_ctr="LAT_CTR" \ - output_varname_nx_of_t7_on_t7g="NX" \ - output_varname_ny_of_t7_on_t7g="NY" \ - output_varname_halo_width_on_t7g="NHW" \ - output_varname_stretch_factor="STRETCH_FAC" \ - output_varname_istart_of_t7_with_halo_on_t6sg="ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - output_varname_iend_of_t7_with_halo_on_t6sg="IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - output_varname_jstart_of_t7_with_halo_on_t6sg="JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - output_varname_jend_of_t7_with_halo_on_t6sg="JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" + verbose="${VERBOSE}" \ + outvarname_lon_of_t7_ctr="LON_CTR" \ + outvarname_lat_of_t7_ctr="LAT_CTR" \ + outvarname_nx_of_t7_on_t7g="NX" \ + outvarname_ny_of_t7_on_t7g="NY" \ + outvarname_halo_width_on_t7g="NHW" \ + outvarname_stretch_factor="STRETCH_FAC" \ + outvarname_istart_of_t7_with_halo_on_t6sg="ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ + outvarname_iend_of_t7_with_halo_on_t6sg="IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ + outvarname_jstart_of_t7_with_halo_on_t6sg="JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ + outvarname_jend_of_t7_with_halo_on_t6sg="JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" # #----------------------------------------------------------------------- # @@ -1881,17 +1929,17 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then halo_width="${ESGgrid_WIDE_HALO_WIDTH}" \ delx="${ESGgrid_DELX}" \ dely="${ESGgrid_DELY}" \ - output_varname_lon_ctr="LON_CTR" \ - output_varname_lat_ctr="LAT_CTR" \ - output_varname_nx="NX" \ - output_varname_ny="NY" \ - output_varname_pazi="PAZI" \ - output_varname_halo_width="NHW" \ - output_varname_stretch_factor="STRETCH_FAC" \ - output_varname_del_angle_x_sg="DEL_ANGLE_X_SG" \ - output_varname_del_angle_y_sg="DEL_ANGLE_Y_SG" \ - output_varname_neg_nx_of_dom_with_wide_halo="NEG_NX_OF_DOM_WITH_WIDE_HALO" \ - output_varname_neg_ny_of_dom_with_wide_halo="NEG_NY_OF_DOM_WITH_WIDE_HALO" + outvarname_lon_ctr="LON_CTR" \ + outvarname_lat_ctr="LAT_CTR" \ + outvarname_nx="NX" \ + outvarname_ny="NY" \ + outvarname_pazi="PAZI" \ + outvarname_halo_width="NHW" \ + outvarname_stretch_factor="STRETCH_FAC" \ + outvarname_del_angle_x_sg="DEL_ANGLE_X_SG" \ + outvarname_del_angle_y_sg="DEL_ANGLE_Y_SG" \ + outvarname_neg_nx_of_dom_with_wide_halo="NEG_NX_OF_DOM_WITH_WIDE_HALO" \ + outvarname_neg_ny_of_dom_with_wide_halo="NEG_NY_OF_DOM_WITH_WIDE_HALO" fi # @@ -2027,12 +2075,8 @@ fi # #----------------------------------------------------------------------- # -check_var_valid_value "WRITE_DOPOST" "valid_vals_WRITE_DOPOST" -# -# Set WRITE_DOPOST to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -WRITE_DOPOST=$(boolify $WRITE_DOPOST) +check_var_valid_value "WRITE_DOPOST" "valid_vals_BOOLEAN" +WRITE_DOPOST=$(boolify "${WRITE_DOPOST}") if [ "$WRITE_DOPOST" = "TRUE" ] ; then @@ -2046,11 +2090,11 @@ SUB_HOURLY_POST is NOT available with Inline Post yet." fi fi -check_var_valid_value "QUILTING" "valid_vals_QUILTING" -QUILTING=$(boolify $QUILTING) +check_var_valid_value "QUILTING" "valid_vals_BOOLEAN" +QUILTING=$(boolify "$QUILTING") -check_var_valid_value "PRINT_ESMF" "valid_vals_PRINT_ESMF" -PRINT_ESMF=$(boolify $PRINT_ESMF) +check_var_valid_value "PRINT_ESMF" "valid_vals_BOOLEAN" +PRINT_ESMF=$(boolify "${PRINT_ESMF}") # #----------------------------------------------------------------------- @@ -2166,6 +2210,27 @@ GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # #----------------------------------------------------------------------- # +# Get the list of constants and their values. The result is saved in +# the variable "constant_defns". This will be written to the experiment's +# variable defintions file later below. +# +#----------------------------------------------------------------------- +# +print_info_msg " +Creating list of constants..." + +get_bash_file_contents fp="$USHDIR/${CONSTANTS_FN}" \ + outvarname_contents="constant_defns" + +print_info_msg "$DEBUG" " +The variable \"constant_defns\" containing definitions of various +constants is set as follows: + +${constant_defns} +" +# +#----------------------------------------------------------------------- +# # Get the list of primary experiment variables and their default values # from the default experiment configuration file (EXPT_DEFAULT_CONFIG_FN). # By "primary", we mean those variables that are defined in the default @@ -2180,7 +2245,7 @@ print_info_msg " Creating list of default experiment variable definitions..." get_bash_file_contents fp="$USHDIR/${EXPT_DEFAULT_CONFIG_FN}" \ - output_varname_contents="default_var_defns" + outvarname_contents="default_var_defns" print_info_msg "$DEBUG" " The variable \"default_var_defns\" containing default values of primary @@ -2294,9 +2359,9 @@ var_name = \"${var_name}\"" else var_value="" - printf -v "var_value" "${escbksl_nl_or_null}" + printf -v "var_value" "%s${escbksl_nl_or_null}" "" for (( i=0; i<${num_elems}; i++ )); do - printf -v "var_value" "${var_value}\"${array[$i]}\" ${escbksl_nl_or_null}" + printf -v "var_value" "%s${escbksl_nl_or_null}" "${var_value}\"${array[$i]}\" " done rhs="( ${var_value})" @@ -2323,7 +2388,7 @@ Setting its value in the variable definitions file to an empty string." # to the list of all variable definitions. # var_defn="${var_name}=$rhs" - printf -v "var_defns" "${var_defns}${var_defn}\n" + printf -v "var_defns" "%s\n" "${var_defns}${var_defn}" # # If var_name is empty, then a variable name was not found on the current # line in default_var_defns. In this case, print out a warning and move @@ -2366,6 +2431,16 @@ var_defns_file_contents="\ #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 1: +# This section contains definitions of the various constants defined in +# the file ${CONSTANTS_FN}. +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# +${constant_defns} +# +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# Section 2: # This section contains (most of) the primary experiment variables, i.e. # those variables that are defined in the default configuration file # (${EXPT_DEFAULT_CONFIG_FN}) and that can be reset via the user-specified @@ -2388,7 +2463,7 @@ var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- #----------------------------------------------------------------------- -# Section 2: +# Section 3: # This section defines variables that have been derived from the primary # set of experiment variables above (we refer to these as \"derived\" or # \"secondary\" variables). @@ -2531,9 +2606,6 @@ SDF_USES_THOMPSON_MP='${SDF_USES_THOMPSON_MP}' # GTYPE='$GTYPE' TILE_RGNL='${TILE_RGNL}' -NH0='${NH0}' -NH3='${NH3}' -NH4='${NH4}' LON_CTR='${LON_CTR}' LAT_CTR='${LAT_CTR}' diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 69c08984e1..c3b0fe8f9d 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,9 +1,11 @@ # +# Source file containing useful constants. +# +. ./constants.sh +# # Define valid values for various global experiment/workflow variables. # valid_vals_RUN_ENVIR=("nco" "community") -valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY" "GAEA") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") @@ -46,40 +48,13 @@ valid_vals_CCPP_PHYS_SUITE=( \ valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") -valid_vals_USE_USER_STAGED_EXTRN_FILES=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2" "netcdf") valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2" "netcdf") valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid") valid_vals_PREEXISTING_DIR_METHOD=("delete" "rename" "quit") valid_vals_GTYPE=("regional") valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_latlon") -valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_MAKE_OROG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_MAKE_SFC_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_RUN_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_WRITE_DOPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_GRIDSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_POINTSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_ENSGRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_ENSPOINT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DOT_OR_USCORE=("." "_") -valid_vals_NOMADS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_NOMADS_file_type=("GRIB2" "grib2" "NEMSIO" "nemsio") -valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_CUSTOM_POST_CONFIG_FILE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_CRTM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_LSM_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_FVCOM_WCSTART=("warm" "WARM" "cold" "COLD") valid_vals_COMPILER=("intel" "gnu") -valid_vals_SUB_HOURLY_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") -valid_vals_USE_MERRA_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From bc77ad6f2e3975c5334d17b83ccc01779606b503 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Wed, 1 Jun 2022 15:17:26 -0600 Subject: [PATCH 180/203] Bug fix for calculation of number of time steps for a given WE2E test. (#782) ## DESCRIPTION OF CHANGES: This PR fixes a bug in the way the number of time steps (`num_time_steps` in `get_WE2Etest_names_subdirs_descs.sh`) is calculated for each WE2E test when creating the CSV file containing information about the available WE2E tests. ## TESTS CONDUCTED: Reran the `run_WE2E_tests.sh` script (which in turn calls `get_WE2Etest_names_subdirs_descs.sh`), and a CSV file containing the adjusted numbers was successfully created. ## DOCUMENTATION: None needed. --- tests/WE2E/get_WE2Etest_names_subdirs_descs.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 57262e9261..c2a6f72fc6 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -1337,9 +1337,12 @@ ${test_desc}${stripped_line} " num_grid_pts=$(( nx*ny )) # # Calculate the number of time steps for the test. Note that FCST_LEN_HRS -# is in units of hours while dta is in units of seconds. +# is in units of hours while dta is in units of seconds. Also, the factor +# dta - 1 in the numerator is to cause the division to round up to the +# nearest integer (adding the denominator minus one to the numerator will +# make this happen). # - num_time_steps=$(( FCST_LEN_HRS*3600/dta + 1 )) + num_time_steps=$(( (FCST_LEN_HRS*3600 + dta - 1)/dta )) # # Calculate the absolute cost of the test. # @@ -1403,7 +1406,7 @@ ${test_desc}${stripped_line} " num_grid_pts=$(( nx*ny )) fcst_len_hrs="6" - num_time_steps=$(( fcst_len_hrs*3600/dta+ 1 )) + num_time_steps=$(( (fcst_len_hrs*3600 + dta - 1)/dta )) abs_cost_ref=$(( num_grid_pts*num_time_steps )) for (( i=0; i<=$((num_prim_tests-1)); i++ )); do From 429a174415ce4fcb0c0f6229c6d63addec2a91fa Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:03:53 -0600 Subject: [PATCH 181/203] Bugfix: Pass domain name to python plotting scripts (#783) ## DESCRIPTION OF CHANGES: PR #[763](https://github.com/ufs-community/regional_workflow/pull/763) introduced the experiment variable `POST_OUTPUT_DOMAIN_NAME` and used it to modify the names of the UPP output files. This PR adds `POST_OUTPUT_DOMAIN_NAME` as an argument to the python plotting scripts to enable those scripts to form the proper UPP output file names to read in. It also modifies the batch scripts (`qsub_job.sh`, `qsub_job_diff.sh`, `sq_job.sh`, and `sq_job_diff.sh`) to add the new argument to the calls to the python plotting scripts that are called. ## TESTS CONDUCTED: Both python plotting scripts `plot_allvars.py` and `plot_allvars_diff.py` were run on Hera on preexisting UPP output files and completed successfully. The batch scripts `sq_job.sh` and `sq_job_diff.sh` were also run successfully on Hera (`qsub_job.sh` and `qsub_job_diff.sh` were not run on e.g. Cheyenne). ## DEPENDENCIES: PR # in `ufs-srweather-app`. ## DOCUMENTATION: Modifications to the documentation are in PR #. ## ISSUE: Partially resolves issue #781 (still need to introduce tests in `regional_workflow` that run the plotting scripts). ## CONTRIBUTORS: @mark-a-potts pointed out the problem and first made the necessary fixes in the `NOAA-EPIC` fork of `regional_workflow`; @gsketefian introduced those changes to this PR and made edits to the comments and documentation. --- ush/Python/plot_allvars.py | 14 +++++++++---- ush/Python/plot_allvars_diff.py | 17 +++++++++++----- ush/Python/qsub_job.sh | 9 +++++++-- ush/Python/qsub_job_diff.sh | 15 ++++++++++---- ush/Python/sq_job.sh | 11 +++++++--- ush/Python/sq_job_diff.sh | 15 ++++++++++---- ush/config_defaults.sh | 36 ++++++++++++++++++--------------- ush/setup.sh | 18 ++++++++++++----- 8 files changed, 92 insertions(+), 43 deletions(-) diff --git a/ush/Python/plot_allvars.py b/ush/Python/plot_allvars.py index dd4e29786f..97a9afdbf1 100755 --- a/ush/Python/plot_allvars.py +++ b/ush/Python/plot_allvars.py @@ -26,12 +26,16 @@ # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide +# 7. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecast and in constructing the names +# of the post output files. # -# To create plots for forecast hours 20-24 from 5/7 00Z -# cycle with hourly output: +# To create plots for a forecast on the RRFS_CONUS_25km +# grid for hours 20-24 from the 5/7 00Z cycle with +# hourly output: # python plot_allvars.py 2020050700 20 24 1 \ # /path/to/expt_dirs/experiment/name \ -# /path/to/base/cartopy/maps +# /path/to/base/cartopy/maps RRFS_CONUS_25km # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -227,6 +231,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): parser.add_argument("Forecast hour increment") parser.add_argument("Path to experiment directory") parser.add_argument("Path to base directory of cartopy shapefiles") +parser.add_argument("Name of native domain used in forecast (and in constructing post file names)") args = parser.parse_args() # Read date/time, forecast hour, and directory paths from command line @@ -252,6 +257,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): EXPT_DIR = str(sys.argv[5]) CARTOPY_DIR = str(sys.argv[6]) +POST_OUTPUT_DOMAIN_NAME = str(sys.argv[7]).lower() # Loop over forecast hours for fhr in fhours: @@ -261,7 +267,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): vtime = ndate(itime,int(fhr)) # Define the location of the input file - data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslev.f'+fhour+'.'+POST_OUTPUT_DOMAIN_NAME+'.grib2') # Get the lats and lons grids = [data1] diff --git a/ush/Python/plot_allvars_diff.py b/ush/Python/plot_allvars_diff.py index 9eddb5e109..fe4cb56ebc 100755 --- a/ush/Python/plot_allvars_diff.py +++ b/ush/Python/plot_allvars_diff.py @@ -29,12 +29,17 @@ # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp # -More information regarding files needed to setup # display maps in Cartopy, see SRW App Users' Guide +# 8. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecasts and in constructing the names +# of the post output files. This must be the same +# for both forecasts. # -# To create plots for forecast hours 20-24 from 5/7 00Z -# cycle with hourly output: +# To create difference plots for two forecasts on the +# RRFS_CONUS_25km grid for hours 20-24 from the 5/7 +# 00Z cycle with hourly output: # python plot_allvars_diff.py 2020050700 20 24 1 \ # /path/to/expt_dir_1 /path/to/expt_dir_2 \ -# /path/to/base/cartopy/maps +# /path/to/base/cartopy/maps RRFS_CONUS_25km # # The variable domains in this script can be set to either # 'conus' for a CONUS map or 'regional' where the map @@ -231,6 +236,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): parser.add_argument("Path to experiment 1 directory") parser.add_argument("Path to experiment 2 directory") parser.add_argument("Path to base directory of cartopy shapefiles") +parser.add_argument("Name of native domain used in forecasts (and in constructing post file names)") args = parser.parse_args() # Read date/time, forecast hour, and directory paths from command line @@ -257,6 +263,7 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): EXPT_DIR_1 = str(sys.argv[5]) EXPT_DIR_2 = str(sys.argv[6]) CARTOPY_DIR = str(sys.argv[7]) +POST_OUTPUT_DOMAIN_NAME = str(sys.argv[8]).lower() # Loop over forecast hours for fhr in fhours: @@ -267,8 +274,8 @@ def rotate_wind(true_lat,lov_lon,earth_lons,uin,vin,proj,inverse=False): # Define the location of the input files - data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') - data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslevf'+fhour+'.tm00.grib2') + data1 = pygrib.open(EXPT_DIR_1+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslev.f'+fhour+'.'+POST_OUTPUT_DOMAIN_NAME+'.grib2') + data2 = pygrib.open(EXPT_DIR_2+'/'+ymdh+'/postprd/rrfs.t'+cyc+'z.prslev.f'+fhour+'.'+POST_OUTPUT_DOMAIN_NAME+'.grib2') # Get the lats and lons grids = [data1, data2] diff --git a/ush/Python/qsub_job.sh b/ush/Python/qsub_job.sh index 25e8814189..4640208767 100755 --- a/ush/Python/qsub_job.sh +++ b/ush/Python/qsub_job.sh @@ -35,7 +35,7 @@ export FCST_END=${FCST_LEN_HRS} export FCST_INC=3 # Usage statement: Make sure all the necessary modules can be imported. -# Five command line arguments are needed: +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour in HHH format # 3. Ending forecast hour in HHH format @@ -46,5 +46,10 @@ export FCST_INC=3 # 6. CARTOPY_DIR: Base directory of cartopy shapefiles # -File structure should be: # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp +# 7. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecast and in constructing the names +# of the post output files. -python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR} ${SHAPE_FILES} + +python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} \ + ${EXPTDIR} ${SHAPE_FILES} ${POST_OUTPUT_DOMAIN_NAME} diff --git a/ush/Python/qsub_job_diff.sh b/ush/Python/qsub_job_diff.sh index e0a6383863..c3f5e4fba6 100755 --- a/ush/Python/qsub_job_diff.sh +++ b/ush/Python/qsub_job_diff.sh @@ -28,7 +28,7 @@ ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics #Cheyenne: SHAPE_FILES=/glade/p/ral/jntp/UFS_SRW_app/tools/NaturalEarth -export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR1}/var_defns.sh" source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export FCST_START=3 @@ -36,7 +36,7 @@ export FCST_END=${FCST_LEN_HRS} export FCST_INC=3 # Usage statement: Make sure all the necessary modules can be imported. -# Seven command line arguments are needed: +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour in HHH format # 3. Ending forecast hour in HHH format @@ -50,5 +50,12 @@ export FCST_INC=3 # 7. CARTOPY_DIR: Base directory of cartopy shapefiles # -File structure should be: # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp - -python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR1} ${EXPTDIR2} ${SHAPE_FILES} +# 8. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecasts and in constructing the names +# of the post output files. This must be the same +# for both forecasts. + +python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} \ + ${EXPTDIR1} ${EXPTDIR2} \ + ${SHAPE_FILES} \ + ${POST_OUTPUT_DOMAIN_NAME} diff --git a/ush/Python/sq_job.sh b/ush/Python/sq_job.sh index 1c594efd46..d4a5947462 100755 --- a/ush/Python/sq_job.sh +++ b/ush/Python/sq_job.sh @@ -57,8 +57,8 @@ export FCST_START=6 export FCST_END=${FCST_LEN_HRS} export FCST_INC=6 -# Usage statement: Make sure all the necessary modules can be imported. -# Six command line arguments are needed: +# Usage statement: Make sure all the necessary modules can be imported. +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour in HHH format # 3. Ending forecast hour in HHH format @@ -69,5 +69,10 @@ export FCST_INC=6 # 6. CARTOPY_DIR: Base directory of cartopy shapefiles # -File structure should be: # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp +# 7. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecast and in constructing the names +# of the post output files. -python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR} ${SHAPE_FILES} + +python plot_allvars.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} \ + ${EXPTDIR} ${SHAPE_FILES} ${POST_OUTPUT_DOMAIN_NAME} diff --git a/ush/Python/sq_job_diff.sh b/ush/Python/sq_job_diff.sh index 84143ffd86..e7be0595f5 100755 --- a/ush/Python/sq_job_diff.sh +++ b/ush/Python/sq_job_diff.sh @@ -52,15 +52,15 @@ SHAPE_FILES=/scratch2/BMC/det/UFS_SRW_app/v1p0/fix_files/NaturalEarth #Gaea: #SHAPE_FILES=/lustre/f2/pdata/esrl/gsd/ufs/NaturalEarth -export GLOBAL_VAR_DEFNS_FP="${EXPTDIR}/var_defns.sh" +export GLOBAL_VAR_DEFNS_FP="${EXPTDIR1}/var_defns.sh" source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export FCST_START=6 export FCST_END=${FCST_LEN_HRS} export FCST_INC=3 -# Usage statement: Make sure all the necessary modules can be imported. -# Seven command line arguments are needed: +# Usage statement: Make sure all the necessary modules can be imported. +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour # 3. Ending forecast hour @@ -74,5 +74,12 @@ export FCST_INC=3 # 7. CARTOPY_DIR: Base directory of cartopy shapefiles # -File structure should be: # CARTOPY_DIR/shapefiles/natural_earth/cultural/*.shp +# 8. POST_OUTPUT_DOMAIN_NAME: Name of native domain +# used in forecasts and in constructing the names +# of the post output files. This must be the same +# for both forecasts. -python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} ${EXPTDIR1} ${EXPTDIR2} ${SHAPE_FILES} +python plot_allvars_diff.py ${CDATE} ${FCST_START} ${FCST_END} ${FCST_INC} \ + ${EXPTDIR1} ${EXPTDIR2} \ + ${SHAPE_FILES} \ + ${POST_OUTPUT_DOMAIN_NAME} diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 966bf343d2..59a1dafde7 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -419,18 +419,6 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # #----------------------------------------------------------------------- # -# Set output file name. Definitions: -# -# POST_OUTPUT_DOMAIN_NAME: -# Domain name used in naming the output files of run_post by UPP or inline post. -# Output file name: $NET.tHHz.[var_name].f###.$POST_OUTPUT_DOMAIN_NAME.grib2 -# -#----------------------------------------------------------------------- -# -POST_OUTPUT_DOMAIN_NAME="" -# -#----------------------------------------------------------------------- -# # Set forecast parameters. Definitions: # # DATE_FIRST_CYCL: @@ -484,8 +472,11 @@ FCST_LEN_HRS="24" # with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory # # WRITE_DOPOST: -# Flag that determines whether or not to use the INLINE POST option -# When TRUE, force to turn off run_post (RUN_TASK_RUN_POST=FALSE) in setup.sh +# Flag that determines whether or not to use the inline post feature +# [i.e. calling the Unified Post Processor (UPP) from within the weather +# model]. If this is set to "TRUE", the RUN_POST_TN task is deactivated +# (i.e. RUN_TASK_RUN_POST is set to "FALSE") to avoid unnecessary +# computations. # #----------------------------------------------------------------------- # @@ -1757,8 +1748,7 @@ DT_SUBHOURLY_POST_MNTS="00" # #----------------------------------------------------------------------- # -# Set parameters associated with defining a customized post configuration -# file. +# Set parameters for customizing the post-processor (UPP). Definitions: # # USE_CUSTOM_POST_CONFIG_FILE: # Flag that determines whether a user-provided custom configuration file @@ -1773,10 +1763,24 @@ DT_SUBHOURLY_POST_MNTS="00" # used for post-processing. This is only used if CUSTOM_POST_CONFIG_FILE # is set to "TRUE". # +# POST_OUTPUT_DOMAIN_NAME: +# Domain name (in lowercase) used in constructing the names of the output +# files generated by UPP [which is called either by running the RUN_POST_TN +# task or by activating the inline post feature (WRITE_DOPOST set to "TRUE")]. +# The post output files are named as follows: +# +# $NET.tHHz.[var_name].f###.${POST_OUTPUT_DOMAIN_NAME}.grib2 +# +# If using a custom grid, POST_OUTPUT_DOMAIN_NAME must be specified by +# the user. If using a predefined grid, POST_OUTPUT_DOMAIN_NAME defaults +# to PREDEF_GRID_NAME. Note that this variable is first changed to lower +# case before being used to construct the file names. +# #----------------------------------------------------------------------- # USE_CUSTOM_POST_CONFIG_FILE="FALSE" CUSTOM_POST_CONFIG_FP="" +POST_OUTPUT_DOMAIN_NAME="" # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index e15dad5c28..49153fc68a 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1240,9 +1240,6 @@ check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" # # COMOUT_BASEDIR is not used by the workflow in community mode. # -# POST_OUTPUT_DOMAIN_NAME: -# The PREDEF_GRID_NAME is set by default. -# #----------------------------------------------------------------------- # LOGDIR="${EXPTDIR}/log" @@ -1262,8 +1259,20 @@ else COMROOT="" COMOUT_BASEDIR="" fi - +# +#----------------------------------------------------------------------- +# +# +# If POST_OUTPUT_DOMAIN_NAME has not been specified by the user, set it +# to PREDEF_GRID_NAME (which won't be empty if using a predefined grid). +# Then change it to lowercase. Finally, ensure that it does not end up +# getting set to an empty string. +# +#----------------------------------------------------------------------- +# POST_OUTPUT_DOMAIN_NAME="${POST_OUTPUT_DOMAIN_NAME:-${PREDEF_GRID_NAME}}" +POST_OUTPUT_DOMAIN_NAME=$(echo_lowercase ${POST_OUTPUT_DOMAIN_NAME}) + if [ -z "${POST_OUTPUT_DOMAIN_NAME}" ]; then print_err_msg_exit "\ The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) @@ -1273,7 +1282,6 @@ If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME is set to a null string), POST_OUTPUT_DOMAIN_NAME must be set in the SRW App's configuration file (\"${EXPT_CONFIG_FN}\")." fi -POST_OUTPUT_DOMAIN_NAME=$(echo_lowercase ${POST_OUTPUT_DOMAIN_NAME}) # #----------------------------------------------------------------------- # From edba7175a92171fc4f9630e54a2730f27e6dc49b Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 9 Jun 2022 16:42:37 -0400 Subject: [PATCH 182/203] Update scripts to work with the latest hashes of UFS_UTILS and UPP (#775) * update input namelist of chgres_cube * update diag_table templates * update scripts * back to original * specify miniconda version on Jet --- modulefiles/tasks/jet/miniconda_regional_workflow | 4 ++-- scripts/exregional_make_ics.sh | 1 - scripts/exregional_make_lbcs.sh | 1 - ush/config_defaults.sh | 2 +- ush/templates/diag_table.FV3_GFS_2017_gfdlmp | 2 +- ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional | 2 +- ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km | 2 +- ush/templates/diag_table.FV3_GFS_v15p2 | 2 +- ush/templates/diag_table.FV3_GFS_v16 | 2 +- ush/templates/diag_table.FV3_HRRR | 2 +- ush/templates/diag_table.FV3_RRFS_v0 | 2 +- ush/templates/diag_table.FV3_RRFS_v1alpha | 2 +- ush/templates/diag_table.FV3_RRFS_v1beta | 2 +- 13 files changed, 12 insertions(+), 14 deletions(-) diff --git a/modulefiles/tasks/jet/miniconda_regional_workflow b/modulefiles/tasks/jet/miniconda_regional_workflow index 011a832c9b..48de7a99bb 100644 --- a/modulefiles/tasks/jet/miniconda_regional_workflow +++ b/modulefiles/tasks/jet/miniconda_regional_workflow @@ -1,5 +1,5 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module use /contrib/miniconda3/modulefiles +module load miniconda3/4.5.12 setenv SRW_ENV regional_workflow diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index fcfefd6462..7af832c4ce 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -518,7 +518,6 @@ fi # settings=" 'config': { - 'fix_dir_input_grid': ${FIXgsm}, 'fix_dir_target_grid': ${FIXLAM}, 'mosaic_file_target_grid': ${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, 'orog_dir_target_grid': ${FIXLAM}, diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 70ba511763..a98ab354df 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -442,7 +442,6 @@ list file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): # settings=" 'config': { - 'fix_dir_input_grid': ${FIXgsm}, 'fix_dir_target_grid': ${FIXLAM}, 'mosaic_file_target_grid': ${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, 'orog_dir_target_grid': ${FIXLAM}, diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 59a1dafde7..28d4564204 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1921,7 +1921,7 @@ LSM_SPP_TSCALE=( "21600" "21600" "21600" "21600" "21600" "21600" "21600" ) LSM_SPP_LSCALE=( "150000" "150000" "150000" "150000" "150000" "150000" "150000" ) ISEED_LSM_SPP=( "9" ) LSM_SPP_VAR_LIST=( "smc" "vgf" "alb" "sal" "emi" "zol" "stc" ) -LSM_SPP_MAG_LIST=( "0.2" "0.001" "0.001" "0.001" "0.001" "0.001" "0.2" ) +LSM_SPP_MAG_LIST=( "0.017" "0.001" "0.001" "0.001" "0.001" "0.001" "0.2" ) LSM_SPP_EACH_STEP="true" #Sets lndp_each_step=.true. # #----------------------------------------------------------------------- diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp index 1b46eeb99a..8b64d1ee91 100755 --- a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp +++ b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp @@ -140,7 +140,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional index f94509ac7d..dbcfbb1f65 100644 --- a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional +++ b/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional @@ -175,7 +175,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km b/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km index 84acd3caf5..6f32189582 100644 --- a/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km +++ b/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km @@ -169,7 +169,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GFS_v15p2 b/ush/templates/diag_table.FV3_GFS_v15p2 index d4b1af5320..129061171f 100755 --- a/ush/templates/diag_table.FV3_GFS_v15p2 +++ b/ush/templates/diag_table.FV3_GFS_v15p2 @@ -140,7 +140,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_GFS_v16 b/ush/templates/diag_table.FV3_GFS_v16 index d4b1af5320..129061171f 100755 --- a/ush/templates/diag_table.FV3_GFS_v16 +++ b/ush/templates/diag_table.FV3_GFS_v16 @@ -140,7 +140,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_HRRR b/ush/templates/diag_table.FV3_HRRR index fa16a7e107..45d5c87d19 100755 --- a/ush/templates/diag_table.FV3_HRRR +++ b/ush/templates/diag_table.FV3_HRRR @@ -152,7 +152,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v0 b/ush/templates/diag_table.FV3_RRFS_v0 index 99b7652910..a3cb733d5c 100644 --- a/ush/templates/diag_table.FV3_RRFS_v0 +++ b/ush/templates/diag_table.FV3_RRFS_v0 @@ -152,7 +152,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v1alpha b/ush/templates/diag_table.FV3_RRFS_v1alpha index 0bc7e358d4..b97d67f61a 100755 --- a/ush/templates/diag_table.FV3_RRFS_v1alpha +++ b/ush/templates/diag_table.FV3_RRFS_v1alpha @@ -152,7 +152,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 diff --git a/ush/templates/diag_table.FV3_RRFS_v1beta b/ush/templates/diag_table.FV3_RRFS_v1beta index 03ce373f4c..00690a5498 100755 --- a/ush/templates/diag_table.FV3_RRFS_v1beta +++ b/ush/templates/diag_table.FV3_RRFS_v1beta @@ -152,7 +152,7 @@ "gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwatclm", "fv3_history2d", "all", .false., "none", 2 +"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 From 270ff1ec540df64d395fa5a35c23b4d4035665ce Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Fri, 10 Jun 2022 16:48:16 -0500 Subject: [PATCH 183/203] adding new e2e tests (#792) --- ..._13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh | 25 +++++++++++++++++++ ...cs_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 25 +++++++++++++++++++ ..._25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh | 25 +++++++++++++++++++ ...cs_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 25 +++++++++++++++++++ ...S_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh | 25 +++++++++++++++++++ ...cs_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh | 25 +++++++++++++++++++ 6 files changed, 150 insertions(+) create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh create mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh new file mode 100644 index 0000000000..f85ed71e1f --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the HRRR +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..b759b00528 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_13km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_13km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh new file mode 100644 index 0000000000..d8da153c16 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the HRRR +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..2b342d9eef --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_25km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh new file mode 100644 index 0000000000..0435df0a2f --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the HRRR +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh new file mode 100644 index 0000000000..41f4d86a34 --- /dev/null +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -0,0 +1,25 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test is to ensure that the workflow running in community mode +# completes successfully on the RRFS_CONUS_3km grid using the RRFS_v1beta +# physics suite with ICs and LBCs derived from the FV3GFS. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_3km" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" From 8be67a4ed56111be0d54aa00b7dcc1a728972aaf Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Thu, 23 Jun 2022 08:29:55 -0400 Subject: [PATCH 184/203] [develop] Update scripts to run with the latest hash of ufs weather model (#796) * update new variables introduced in uwm * update python script * update template for HRRR suite --- ush/config_defaults.sh | 7 +------ ush/config_defaults.yaml | 9 ++------- ush/generate_FV3LAM_wflow.py | 2 +- ush/generate_FV3LAM_wflow.sh | 2 +- ush/setup.py | 5 ++++- ush/setup.sh | 3 +++ ush/templates/FV3.input.yml | 5 +++++ ush/templates/input.nml.FV3 | 3 ++- 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 28d4564204..eb04c68be1 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1901,10 +1901,6 @@ ISEED_SPP=( "4" "5" "6" "7" "8" ) # SPP in LSM schemes is handled in the &nam_sfcperts namelist block # instead of in &nam_sppperts, where all other SPP is implemented. # -# The default perturbation frequency is determined by the fhcyc namelist -# entry. Since that parameter is set to zero in the SRW App, use -# LSM_SPP_EACH_STEP to perturb every time step. -# # Perturbations to soil moisture content (SMC) are only applied at the # first time step. # @@ -1916,13 +1912,12 @@ ISEED_SPP=( "4" "5" "6" "7" "8" ) # are shown below. In addition, only one unique iseed value is allowed # at the moment, and is used for each pattern. # -DO_LSM_SPP="false" #If true, sets lndp_type=2 +DO_LSM_SPP="false" #If true, sets lndp_type=2, lndp_model_type=2 LSM_SPP_TSCALE=( "21600" "21600" "21600" "21600" "21600" "21600" "21600" ) LSM_SPP_LSCALE=( "150000" "150000" "150000" "150000" "150000" "150000" "150000" ) ISEED_LSM_SPP=( "9" ) LSM_SPP_VAR_LIST=( "smc" "vgf" "alb" "sal" "emi" "zol" "stc" ) LSM_SPP_MAG_LIST=( "0.017" "0.001" "0.001" "0.001" "0.001" "0.001" "0.2" ) -LSM_SPP_EACH_STEP="true" #Sets lndp_each_step=.true. # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 56b89bb117..6498ecf013 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1888,10 +1888,6 @@ ISEED_SPP: [ "4", "4", "4", "4", "4" ] # SPP in LSM schemes is handled in the &nam_sfcperts namelist block # instead of in &nam_sppperts, where all other SPP is implemented. # -# The default perturbation frequency is determined by the fhcyc namelist -# entry. Since that parameter is set to zero in the SRW App, use -# LSM_SPP_EACH_STEP to perturb every time step. -# # Perturbations to soil moisture content (SMC) are only applied at the # first time step. # @@ -1903,13 +1899,12 @@ ISEED_SPP: [ "4", "4", "4", "4", "4" ] # are shown below. In addition, only one unique iseed value is allowed # at the moment, and is used for each pattern. # -DO_LSM_SPP: "FALSE" #If true, sets lndp_type=2 +DO_LSM_SPP: "FALSE" #If true, sets lndp_type=2, lndp_model_type=2 LSM_SPP_TSCALE: [ "21600", "21600", "21600", "21600", "21600", "21600", "21600" ] LSM_SPP_LSCALE: [ "150000", "150000", "150000", "150000", "150000", "150000", "150000" ] ISEED_LSM_SPP: [ "9" ] LSM_SPP_VAR_LIST: [ "smc", "vgf", "alb", "sal", "emi", "zol", "stc" ] -LSM_SPP_MAG_LIST: [ "0.2", "0.001", "0.001", "0.001", "0.001", "0.001", "0.2" ] -LSM_SPP_EACH_STEP: "TRUE" #Sets lndp_each_step=.true. +LSM_SPP_MAG_LIST: [ "0.017", "0.001", "0.001", "0.001", "0.001", "0.001", "0.2" ] # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 2391340c40..ad1e205d45 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -737,7 +737,6 @@ def generate_FV3LAM_wflow(): 'n_var_spp': N_VAR_SPP, 'n_var_lndp': N_VAR_LNDP, 'lndp_type': LNDP_TYPE, - 'lndp_each_step': LSM_SPP_EACH_STEP, 'fhcyc': FHCYC_LSM_SPP_OR_NOT } # @@ -865,6 +864,7 @@ def generate_FV3LAM_wflow(): if DO_LSM_SPP: nam_sfcperts_dict = { 'lndp_type': LNDP_TYPE, + 'lndp_model_type': LNDP_MODEL_TYPE, 'lndp_tau': LSM_SPP_TSCALE, 'lndp_lscale': LSM_SPP_LSCALE, 'iseed_lndp': ISEED_LSM_SPP, diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 8af7497b03..d9f2299139 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -789,7 +789,6 @@ settings="\ 'n_var_spp': ${N_VAR_SPP}, 'n_var_lndp': ${N_VAR_LNDP}, 'lndp_type': ${LNDP_TYPE}, - 'lndp_each_step': ${LSM_SPP_EACH_STEP}, 'fhcyc': ${FHCYC_LSM_SPP_OR_NOT}, }" # @@ -932,6 +931,7 @@ settings="$settings if [ "${DO_LSM_SPP}" = "TRUE" ]; then settings="$settings 'lndp_type': ${LNDP_TYPE}, + 'lndp_model_type': ${LNDP_MODEL_TYPE}, 'lndp_tau': [ $( printf "%s, " "${LSM_SPP_TSCALE[@]}" ) ], 'lndp_lscale': [ $( printf "%s, " "${LSM_SPP_LSCALE[@]}" ) ], 'iseed_lndp': [ $( printf "%s, " "${ISEED_LSM_SPP[@]}" ) ], diff --git a/ush/setup.py b/ush/setup.py index 2e6cb912c4..06bf3dd535 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -167,13 +167,15 @@ def setup(): # #----------------------------------------------------------------------- # - global N_VAR_LNDP, LNDP_TYPE, FHCYC_LSM_SPP_OR_NOT + global N_VAR_LNDP, LNDP_TYPE, LNDP_MODEL_TYPE, FHCYC_LSM_SPP_OR_NOT N_VAR_LNDP=0 LNDP_TYPE=0 + LNDP_MODEL_TYPE=0 FHCYC_LSM_SPP_OR_NOT=0 if DO_LSM_SPP: N_VAR_LNDP=len(LSM_SPP_VAR_LIST) LNDP_TYPE=2 + LNDP_MODEL_TYPE=2 FHCYC_LSM_SPP_OR_NOT=999 # #----------------------------------------------------------------------- @@ -2163,6 +2165,7 @@ def setup(): N_VAR_SPP='{N_VAR_SPP}' N_VAR_LNDP='{N_VAR_LNDP}' LNDP_TYPE='{LNDP_TYPE}' + LNDP_MODEL_TYPE='{LNDP_MODEL_TYPE}' FHCYC_LSM_SPP_OR_NOT='{FHCYC_LSM_SPP_OR_NOT}' """ diff --git a/ush/setup.sh b/ush/setup.sh index 49153fc68a..108080db23 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -286,10 +286,12 @@ fi # N_VAR_LNDP=0 LNDP_TYPE=0 +LNDP_MODEL_TYPE=0 FHCYC_LSM_SPP_OR_NOT=0 if [ "${DO_LSM_SPP}" = "TRUE" ]; then N_VAR_LNDP=${#LSM_SPP_VAR_LIST[@]} LNDP_TYPE=2 + LNDP_MODEL_TYPE=2 FHCYC_LSM_SPP_OR_NOT=999 fi # @@ -2808,6 +2810,7 @@ PE_MEMBER01='${PE_MEMBER01}' N_VAR_SPP='${N_VAR_SPP}' N_VAR_LNDP='${N_VAR_LNDP}' LNDP_TYPE='${LNDP_TYPE}' +LNDP_MODEL_TYPE='${LNDP_MODEL_TYPE}' FHCYC_LSM_SPP_OR_NOT='${FHCYC_LSM_SPP_OR_NOT}' " # diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index f9256bc080..7b5dab6d77 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -30,6 +30,7 @@ FV3_RRFS_v1alpha: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 + iopt_trs: 2 lsm: 2 lsoil_lsm: 4 @@ -64,6 +65,7 @@ FV3_HRRR: iovr: 3 lsm: 3 lsoil_lsm: 9 + sfclay_compute_flux: True FV3_GFS_2017_gfdlmp: atmos_model_nml: @@ -182,6 +184,7 @@ FV3_GFS_2017_gfdlmp_regional: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 + iopt_trs: 2 lgfdlmprad: True lheatstrg: False lndp_type: 0 @@ -256,6 +259,7 @@ FV3_GFS_v15p2: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 + iopt_trs: 2 ldiag_ugwp: False lgfdlmprad: True lradar: !!python/none @@ -310,6 +314,7 @@ FV3_GFS_v15_thompson_mynn_lam3km: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 + iopt_trs: 2 iovr: 3 ldiag_ugwp: false lgfdlmprad: false diff --git a/ush/templates/input.nml.FV3 b/ush/templates/input.nml.FV3 index f94d00cbeb..e33c09522c 100644 --- a/ush/templates/input.nml.FV3 +++ b/ush/templates/input.nml.FV3 @@ -47,7 +47,7 @@ &fms_nml clock_grain = 'ROUTINE' - domains_stack_size = 3000000 + domains_stack_size = 5000000 print_memory_usage = .false. / @@ -184,6 +184,7 @@ random_clds = .false. redrag = .true. satmedmf = .false. + sfclay_compute_flux = .false. shal_cnv = .false. swhtr = .true. trans_trac = .true. From f0f369f79414047d8d0f5446262a97b2c94a437d Mon Sep 17 00:00:00 2001 From: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Date: Fri, 24 Jun 2022 10:00:14 -0600 Subject: [PATCH 185/203] [develop] Update MET/METplus versions and configuration files (#779) * remove unused module loads for MET * update met version in machine files * changes to pointstat/pb2nc bufr mapping * add NA to retain bufr variables without qc markers * updated bufr variable list * change bufr var list * Change versions to MET 10.1.1 and METplus 4.1.1 --- modulefiles/tasks/cheyenne/run_vx.local | 2 -- modulefiles/tasks/gaea/run_vx.local | 2 -- modulefiles/tasks/hera/run_vx.local | 2 -- modulefiles/tasks/wcoss_dell_p3/run_vx.local | 3 --- ush/machine/cheyenne.sh | 4 ++-- ush/machine/hera.sh | 4 ++-- ush/machine/orion.sh | 4 ++-- ush/machine/wcoss_dell_p3.sh | 4 ++-- ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf | 7 +++---- ush/templates/parm/metplus/EnsembleStat_upper_air.conf | 7 +++---- ush/templates/parm/metplus/PointStat_conus_sfc.conf | 7 +++---- ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf | 7 +++---- ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf | 7 +++---- ush/templates/parm/metplus/PointStat_upper_air.conf | 7 +++---- ush/templates/parm/metplus/PointStat_upper_air_mean.conf | 7 +++---- ush/templates/parm/metplus/PointStat_upper_air_prob.conf | 7 +++---- 16 files changed, 32 insertions(+), 49 deletions(-) diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local index 8fb3be36da..2f92a39e57 100644 --- a/modulefiles/tasks/cheyenne/run_vx.local +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -1,4 +1,2 @@ #%Module module load pylib_regional_workflow -module use /glade/p/ral/jntp/MET/MET_releases/modulefiles -module load met/10.0.0 diff --git a/modulefiles/tasks/gaea/run_vx.local b/modulefiles/tasks/gaea/run_vx.local index 929c2011f8..51416ff471 100644 --- a/modulefiles/tasks/gaea/run_vx.local +++ b/modulefiles/tasks/gaea/run_vx.local @@ -3,5 +3,3 @@ module use -a /contrib/anaconda/modulefiles module load intel/18.0.5.274 module load anaconda/latest -module use -a /contrib/met/modulefiles/ -module load met/10.0.0 diff --git a/modulefiles/tasks/hera/run_vx.local b/modulefiles/tasks/hera/run_vx.local index 929c2011f8..51416ff471 100644 --- a/modulefiles/tasks/hera/run_vx.local +++ b/modulefiles/tasks/hera/run_vx.local @@ -3,5 +3,3 @@ module use -a /contrib/anaconda/modulefiles module load intel/18.0.5.274 module load anaconda/latest -module use -a /contrib/met/modulefiles/ -module load met/10.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_vx.local b/modulefiles/tasks/wcoss_dell_p3/run_vx.local index 35809d9bf6..3806a6c4c2 100644 --- a/modulefiles/tasks/wcoss_dell_p3/run_vx.local +++ b/modulefiles/tasks/wcoss_dell_p3/run_vx.local @@ -1,6 +1,3 @@ #%Module module unload netcdf - -module use /gpfs/dell2/emc/verification/noscrub/emc.metplus/modulefiles -module load met/10.0.0 diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh index cf80db74f4..2118c9867b 100644 --- a/ush/machine/cheyenne.sh +++ b/ush/machine/cheyenne.sh @@ -61,8 +61,8 @@ RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' RUN_CMD_POST='mpirun -np $nprocs' # MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.0.0"} -METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.0.0"} +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.1.1"} +METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.1.1"} CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index 454a1e9383..f8c9324a97 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -66,8 +66,8 @@ RUN_CMD_FCST="srun" RUN_CMD_POST="srun" # MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.0.0"} -METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.0.0"} +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.1.1"} +METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.1.1"} CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh index b8032e2412..fe01323b75 100644 --- a/ush/machine/orion.sh +++ b/ush/machine/orion.sh @@ -60,8 +60,8 @@ RUN_CMD_FCST='srun -n ${PE_MEMBER01}' RUN_CMD_POST="srun" # MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.0"} -METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.0.0"} +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.1"} +METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.1.1"} CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 9bf525a353..1794452d16 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -72,8 +72,8 @@ RUN_CMD_FCST='mpirun -l -np ${PE_MEMBER01}' RUN_CMD_POST="mpirun" # MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0"} -METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0"} +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.1.1"} +METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.1.1"} CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index d660664228..d8bb1e54e5 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -53,12 +53,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # False for no time summary, True otherwise PB2NC_TIME_SUMMARY_FLAG = False @@ -93,7 +92,7 @@ ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPSFC_{OBTYPE} PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped -ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #ENSEMBLE_STAT_OBS_QUALITY_EXC = # if True, pb2nc will skip processing a file if the output already exists diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf index 072926a3b1..c3e24b259d 100644 --- a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf +++ b/ush/templates/parm/metplus/EnsembleStat_upper_air.conf @@ -53,12 +53,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # False for no time summary, True otherwise PB2NC_TIME_SUMMARY_FLAG = False @@ -93,7 +92,7 @@ ENSEMBLE_STAT_OUTPUT_PREFIX = {MODEL}_ADPUPA_{OBTYPE} PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped ENSEMBLE_STAT_CONFIG_FILE = {PARM_BASE}/met_config/EnsembleStatConfig_wrapped -ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +ENSEMBLE_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #ENSEMBLE_STAT_OBS_QUALITY_EXC = # if True, pb2nc will skip processing a file if the output already exists diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index b3a2755a58..4dfea57023 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_WIND, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf index 20e2d7392a..98c23a30d2 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf index f28d022441..dc85505ebc 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/ush/templates/parm/metplus/PointStat_upper_air.conf index 6110122c5d..23591bec11 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_WIND, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf index a2b655136c..1acef02f31 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf index 0212f79d42..cedb5ec648 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -69,12 +69,11 @@ PB2NC_QUALITY_MARK_THRESH = 9 PB2NC_PB_REPORT_TYPE = 120, 220, 221, 122, 222, 223, 224, 131, 133, 233, 153, 156, 157, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 # Leave empty to process all -# PB2NC_OBS_BUFR_VAR_LIST = POB, QOB, TOB, ZOB, UOB, VOB, PMO, TOCC, TDO, HOVI, D_DPT, D_WDIR, D_WIND, D_RH, D_MIXR, D_PRMSL -PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, TDO, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND +PB2NC_OBS_BUFR_VAR_LIST = PMO, ZOB, TOB, D_DPT, QOB, UOB, VOB, PWO, TOCC, D_RH, HOVI, CEILING, D_PBL, D_CAPE, MXGS, D_WIND, D_PRMSL # Mapping of input BUFR variable names to output variables names. # The default PREPBUFR map, obs_prepbufr_map, is appended to this map. -PB2NC_MET_CONFIG_OVERRIDES = obs_bufr_map = [{ key = "HOVI"; val = "VIS"; },{ key = "PMO"; val = "PRMSL"; },{ key = "TDO"; val = "DPT"; }, { key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }, { key = "TOCC"; val = "TCDC"; }]; +PB2NC_OBS_BUFR_MAP = [{ key = "PWO"; val = "PWAT"; },{ key = "MXGS"; val = "GUST"; }, { key = "CEILING"; val = "CEILING"; }] # For defining the time periods for summarization # False for no time summary, True otherwise @@ -91,7 +90,7 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9 +POINT_STAT_OBS_QUALITY_INC = 0, 1, 2, 3, 9, NA #POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST From 80d59ef25fbef1988365a43e82389b86344e2d5e Mon Sep 17 00:00:00 2001 From: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> Date: Mon, 27 Jun 2022 10:14:47 -0400 Subject: [PATCH 186/203] add a linux.sh machine file (a template) (#798) (#803) Co-authored-by: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> --- ush/machine/linux.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 ush/machine/linux.sh diff --git a/ush/machine/linux.sh b/ush/machine/linux.sh new file mode 100644 index 0000000000..cdf055eb4b --- /dev/null +++ b/ush/machine/linux.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -x + +function file_location() { + + # Return the default location of external model files on disk + + local external_file_fmt external_model location + + external_model=${1} + external_file_fmt=${2} + + case ${external_model} in + + "FV3GFS") + location='/home/username/DATA/UFS/FV3GFS/' + ;; + *) + print_info_msg"\ + External model \'${external_model}\' does not have a default + location on Linux systems. " + ;; + + esac + echo ${location:-} +} + + +EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ + ${EXTRN_MDL_NAME_ICS} \ + ${FV3GFS_FILE_FMT_ICS})} +EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ + ${EXTRN_MDL_NAME_LBCS} \ + ${FV3GFS_FILE_FMT_ICS})} + + System scripts to source to initialize various commands within workflow + scripts (e.g. "module"). +if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then + ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) +fi + + +# Commands to run at the start of each workflow task. +PRE_TASK_CMDS='{ ulimit -a; }' + +# Architecture information +WORKFLOW_MANAGER="none" +NCORES_PER_NODE=${NCORES_PER_NODE:-8} +SCHED=${SCHED:-"none"} + +# UFS SRW App specific paths +FIXgsm=${FIXgsm:-"/home/username/DATA/UFS/fix/fix_am"} +FIXaer=${FIXaer:-"/home/username/DATA/UFS/fix/fix_aer"} +FIXlut=${FIXlut:-"/home/username/DATA/UFS/fix/fix_lut"} +TOPO_DIR=${TOPO_DIR:-"/home/username/DATA/UFS/fix/fix_orog"} +SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/home/username/DATA/UFS/fix/fix_sfc_climo"} + +# Run commands for executables +RUN_CMD_SERIAL="time" +#Run Commands currently differ for GNU/openmpi +RUN_CMD_UTILS='mpirun -n 4' +RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} ' +RUN_CMD_POST='mpirun -n 4 ' + +# MET Installation Locations + From 4dfc9eebbdce3554360cf70b2d40a3151a4d1272 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Mon, 27 Jun 2022 10:18:48 -0600 Subject: [PATCH 187/203] [develop] Rename variables related to GFDLgrid-type grids (#787) ## DESCRIPTION OF CHANGES: For clarity, rename variables related to GFDLgrid-type grids: 1) Rename `GFDLgrid_RES` to `GFDLgrid_NUM_CELLS`. 2) Rename `GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES` to `GFDLgrid_USE_NUM_CELLS_IN_FILENAMES`. 3) Rename WE2E tests on GFDLgrid-type custom grids according to variable renaming above. Related improvements: * In `link_fix.sh`, remove creation of unnecessary symlink when using a GFDLgrid-type grid with `GFDLgrid_USE_NUM_CELLS_IN_FILENAMES` is set to `"FALSE"`. * In WE2E test configuration files that use custom grids, move the definition of the variable `POST_OUTPUT_DOMAIN_NAME` to the beginning of the custom grid definition because it is really the name of the custom native grid, and it's most appropriate to place that at the beginning of the grid definition section. * Reduce `WTIME_MAKE_OROG` from 1 hour to 20 minutes since on Hera, 1 hour causes the task to wait quite a while in the queue. This used to be 20 minutes but was changed to 1 hour because 20 minutes was not sufficient to complete the task on `WCOSS_DELL_P3`. Thus, reset it to 1 hour only in the machine file for `WCOSS_DELL_P3`. * Minor fixes to comments in python plotting scripts. * Edits to other comments. ## TESTS CONDUCTED: Ran the four WE2E tests that use custom grids (listed below) on Hera. All were successful. * `custom_ESGgrid` * `custom_GFDLgrid` * `custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE` (previously `custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE`) * `custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE` (previously `custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE`) ## DEPENDENCIES: PR #[310](https://github.com/ufs-community/ufs-srweather-app/pull/310) in `ufs-srweather-app`. ## DOCUMENTATION: Updates to documentation are in PR #[310](https://github.com/ufs-community/ufs-srweather-app/pull/310) in `ufs-srweather-app`. --- scripts/exregional_make_grid.sh | 6 +- scripts/exregional_make_orog.sh | 2 +- scripts/exregional_run_fcst.sh | 4 +- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 10 +- .../wflow_features/config.custom_ESGgrid.sh | 3 +- .../wflow_features/config.custom_GFDLgrid.sh | 19 ++- ...id_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE.sh} | 17 ++- ...rid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE.sh} | 17 ++- ush/Python/plot_allvars.py | 2 +- ush/Python/plot_allvars_diff.py | 2 +- ush/config_defaults.sh | 112 +++++++++--------- ush/link_fix.sh | 20 ---- ush/machine/wcoss_dell_p3.sh | 4 + ush/set_predef_grid_params.sh | 48 ++++---- ush/setup.sh | 16 +-- ush/valid_param_vals.sh | 2 +- 16 files changed, 130 insertions(+), 154 deletions(-) rename tests/WE2E/test_configs/wflow_features/{config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh => config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE.sh} (82%) rename tests/WE2E/test_configs/wflow_features/{config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh => config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE.sh} (81%) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 3744638c8d..de27902e32 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -246,7 +246,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # Set local variables needed in the call to the executable that generates # a GFDLgrid-type grid. # - nx_t6sg=$(( 2*GFDLgrid_RES )) + nx_t6sg=$(( 2*GFDLgrid_NUM_CELLS )) grid_name="${GRID_GEN_METHOD}" # # Call the executable that generates the grid file. Note that this call @@ -402,8 +402,8 @@ res_equiv=${res_equiv//$'\n'/} #----------------------------------------------------------------------- # if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - if [ "${GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES}" = "TRUE" ]; then - CRES="C${GFDLgrid_RES}" + if [ "${GFDLgrid_USE_NUM_CELLS_IN_FILENAMES}" = "TRUE" ]; then + CRES="C${GFDLgrid_NUM_CELLS}" else CRES="C${res_equiv}" fi diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 1a4d528f24..58e453a97a 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -373,7 +373,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # # Really depends on what EMC wants to do. - res="${GFDLgrid_RES}" + res="${GFDLgrid_NUM_CELLS}" # stretch_fac="${GFDLgrid_STRETCH_FAC}" refine_ratio="${GFDLgrid_REFINE_RATIO}" diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 2f768a6572..8593f9c536 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -167,8 +167,8 @@ create_symlink_to_file target="$target" symlink="$symlink" \ #target="${FIXLAM}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" #if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] && \ # [ "${GRID_GEN_METHOD}" = "GFDLgrid" ] && \ -# [ "${GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES}" = "FALSE" ]; then -# symlink="C${GFDLgrid_RES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" +# [ "${GFDLgrid_USE_NUM_CELLS_IN_FILENAMES}" = "FALSE" ]; then +# symlink="C${GFDLgrid_NUM_CELLS}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" #else # symlink="${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" #fi diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index c2a6f72fc6..39dc68f516 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -409,7 +409,7 @@ function get_WE2Etest_names_subdirs_descs() { \ gfdlgrid_lon_t6_ctr \ gfdlgrid_lat_t6_ctr \ - gfdlgrid_res \ + gfdlgrid_num_cells \ gfdlgrid_stretch_fac \ gfdlgrid_refine_ratio \ gfdlgrid_istart_of_rgnl_dom_on_t6g \ @@ -1245,7 +1245,7 @@ ${test_desc}${stripped_line} " outvarname_gfdlgrid_lon_t6_ctr="gfdlgrid_lon_t6_ctr" \ outvarname_gfdlgrid_lat_t6_ctr="gfdlgrid_lat_t6_ctr" \ outvarname_gfdlgrid_stretch_fac="gfdlgrid_stretch_fac" \ - outvarname_gfdlgrid_res="gfdlgrid_res" \ + outvarname_gfdlgrid_num_cells="gfdlgrid_num_cells" \ outvarname_gfdlgrid_refine_ratio="gfdlgrid_refine_ratio" \ outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="gfdlgrid_istart_of_rgnl_dom_on_t6g" \ outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="gfdlgrid_iend_of_rgnl_dom_on_t6g" \ @@ -1264,7 +1264,7 @@ ${test_desc}${stripped_line} " if [ "${grid_gen_method}" = "GFDLgrid" ]; then gfdlgrid_lon_t6_ctr="${GFDLgrid_LON_T6_CTR}" gfdlgrid_lat_t6_ctr="${GFDLgrid_LAT_T6_CTR}" - gfdlgrid_res="${GFDLgrid_RES}" + gfdlgrid_num_cells="${GFDLgrid_NUM_CELLS}" gfdlgrid_stretch_fac="${GFDLgrid_STRETCH_FAC}" gfdlgrid_refine_ratio="${GFDLgrid_REFINE_RATIO}" gfdlgrid_istart_of_rgnl_dom_on_t6g="${GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G}" @@ -1305,7 +1305,7 @@ ${test_desc}${stripped_line} " set_gridparams_GFDLgrid \ lon_of_t6_ctr="${gfdlgrid_lon_t6_ctr}" \ lat_of_t6_ctr="${gfdlgrid_lat_t6_ctr}" \ - res_of_t6g="${gfdlgrid_res}" \ + res_of_t6g="${gfdlgrid_num_cells}" \ stretch_factor="${gfdlgrid_stretch_fac}" \ refine_ratio_t6g_to_t7g="${gfdlgrid_refine_ratio}" \ istart_of_t7_on_t6g="${gfdlgrid_istart_of_rgnl_dom_on_t6g}" \ @@ -1353,7 +1353,7 @@ ${test_desc}${stripped_line} " # unset gfdlgrid_lon_t6_ctr \ gfdlgrid_lat_t6_ctr \ - gfdlgrid_res \ + gfdlgrid_num_cells \ gfdlgrid_stretch_fac \ gfdlgrid_refine_ratio \ gfdlgrid_istart_of_rgnl_dom_on_t6g \ diff --git a/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh index f78fdf7e0d..86b3ff0b74 100644 --- a/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_ESGgrid.sh @@ -25,6 +25,7 @@ LBC_SPEC_INTVL_HRS="3" # # Define custom grid. # +POST_OUTPUT_DOMAIN_NAME="custom_ESGgrid" GRID_GEN_METHOD="ESGgrid" ESGgrid_LON_CTR="-97.5" @@ -46,8 +47,6 @@ LAYOUT_X="8" LAYOUT_Y="12" BLOCKSIZE="13" -POST_OUTPUT_DOMAIN_NAME="custom_ESGgrid" - QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh index 66604a5491..db148e3ab2 100644 --- a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid.sh @@ -7,14 +7,14 @@ # workflow) of GFDLgrid type. Note that this test sets the workflow # variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_NUM_CELLS_IN_FILENAMES # -# to "TRUE" (which is its default value); see the UFS SRW User's Guide -# for a description of this variable. +# to "TRUE" (which is its default value); see the UFS SRW App's User's +# Guide for a description of this variable. # # The difference between this test and the one named # -# new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE +# new_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE # # is that this one uses almost no stretching by setting the workflow # variable GFDLgrid_STRETCH_FAC very close to 1. Setting it exactly to @@ -42,12 +42,13 @@ LBC_SPEC_INTVL_HRS="3" # # Define custom grid. # +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.0001" # Cannot be exactly 1.0 because then FV3 thinnks it's a global grid. This needs to be fixed in the ufs_weather_model repo. -GFDLgrid_RES="96" +GFDLgrid_NUM_CELLS="96" GFDLgrid_REFINE_RATIO="2" #num_margin_cells_T6_left="9" @@ -55,7 +56,7 @@ GFDLgrid_REFINE_RATIO="2" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" #num_margin_cells_T6_right="9" -#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) +#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" #num_margin_cells_T6_bottom="9" @@ -64,11 +65,11 @@ GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="19" #num_margin_cells_T6_top="9" -#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) +#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_top )) #GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="78" -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" +GFDLgrid_USE_NUM_CELLS_IN_FILENAMES="TRUE" DT_ATMOS="100" @@ -76,8 +77,6 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" -POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" - QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE.sh similarity index 82% rename from tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE.sh index 4c4f0d5c75..ff4d928094 100644 --- a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE.sh @@ -7,10 +7,10 @@ # workflow) of GFDLgrid type. Note that this test sets the workflow # variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_NUM_CELLS_IN_FILENAMES # -# to "FALSE"; see the UFS SRW User's Guide for a description of this -# variable. +# to "FALSE"; see the UFS SRW App's User's Guide for a description of +# this variable. # RUN_ENVIR="community" @@ -31,12 +31,13 @@ LBC_SPEC_INTVL_HRS="3" # # Define custom grid. # +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.5" -GFDLgrid_RES="96" +GFDLgrid_NUM_CELLS="96" GFDLgrid_REFINE_RATIO="2" #num_margin_cells_T6_left="9" @@ -44,7 +45,7 @@ GFDLgrid_REFINE_RATIO="2" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" #num_margin_cells_T6_right="9" -#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) +#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" #num_margin_cells_T6_bottom="9" @@ -52,10 +53,10 @@ GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="10" #num_margin_cells_T6_top="9" -#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) +#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_top )) GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="FALSE" +GFDLgrid_USE_NUM_CELLS_IN_FILENAMES="FALSE" DT_ATMOS="100" @@ -63,8 +64,6 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" -POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" - QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE.sh similarity index 81% rename from tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh rename to tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE.sh index 3e067ee3d2..dc5d79c2ef 100644 --- a/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_TRUE.sh @@ -7,10 +7,10 @@ # workflow) of GFDLgrid type. Note that this test sets the workflow # variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_NUM_CELLS_IN_FILENAMES # -# to "TRUE" (which is its default value); see the UFS SRW User's Guide -# for a description of this variable. +# to "TRUE" (which is its default value); see the UFS SRW App's User's +# Guide for a description of this variable. # RUN_ENVIR="community" @@ -31,12 +31,13 @@ LBC_SPEC_INTVL_HRS="3" # # Define custom grid. # +POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" GRID_GEN_METHOD="GFDLgrid" GFDLgrid_LON_T6_CTR="-97.5" GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.5" -GFDLgrid_RES="96" +GFDLgrid_NUM_CELLS="96" GFDLgrid_REFINE_RATIO="2" #num_margin_cells_T6_left="9" @@ -44,7 +45,7 @@ GFDLgrid_REFINE_RATIO="2" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" #num_margin_cells_T6_right="9" -#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_right )) +#GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_right )) GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" #num_margin_cells_T6_bottom="9" @@ -52,10 +53,10 @@ GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="87" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="10" #num_margin_cells_T6_top="9" -#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_RES - num_margin_cells_T6_top )) +#GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G=$(( GFDLgrid_NUM_CELLS - num_margin_cells_T6_top )) GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="87" -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="TRUE" +GFDLgrid_USE_NUM_CELLS_IN_FILENAMES="TRUE" DT_ATMOS="100" @@ -63,8 +64,6 @@ LAYOUT_X="6" LAYOUT_Y="6" BLOCKSIZE="26" -POST_OUTPUT_DOMAIN_NAME="custom_GFDLgrid" - QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" diff --git a/ush/Python/plot_allvars.py b/ush/Python/plot_allvars.py index 97a9afdbf1..3af564e522 100755 --- a/ush/Python/plot_allvars.py +++ b/ush/Python/plot_allvars.py @@ -9,7 +9,7 @@ # David Wright Org: University of Michigan # # Instructions: Make sure all the necessary modules can be imported. -# Six command line arguments are needed: +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour # 3. Ending forecast hour diff --git a/ush/Python/plot_allvars_diff.py b/ush/Python/plot_allvars_diff.py index fe4cb56ebc..ec85df2e5b 100755 --- a/ush/Python/plot_allvars_diff.py +++ b/ush/Python/plot_allvars_diff.py @@ -9,7 +9,7 @@ # David Wright Org: University of Michigan # # Instructions: Make sure all the necessary modules can be imported. -# Seven command line arguments are needed: +# The following command line arguments are needed: # 1. Cycle date/time in YYYYMMDDHH format # 2. Starting forecast hour # 3. Ending forecast hour diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index eb04c68be1..6790d66861 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -807,20 +807,20 @@ GRID_GEN_METHOD="" # though the model equations are not integrated on (they are integrated # only on the regional grid). # -# * GFDLgrid_RES is the number of grid cells in either one of the two -# horizontal directions x and y on any one of the 6 tiles of the parent -# global cubed-sphere grid. The mapping from GFDLgrid_RES to a nominal -# resolution (grid cell size) for a uniform global grid (i.e. Schmidt -# stretch factor GFDLgrid_STRETCH_FAC set to 1) for several values of -# GFDLgrid_RES is as follows: -# -# GFDLgrid_RES typical cell size -# ------------ ----------------- -# 192 50 km -# 384 25 km -# 768 13 km -# 1152 8.5 km -# 3072 3.2 km +# * GFDLgrid_NUM_CELLS is the number of grid cells in either one of the +# two horizontal directions x and y on any one of the 6 tiles of the +# parent global cubed-sphere grid. The mapping from GFDLgrid_NUM_CELLS +# to a nominal resolution (grid cell size) for a uniform global grid +# (i.e. Schmidt stretch factor GFDLgrid_STRETCH_FAC set to 1) for +# several values of GFDLgrid_NUM_CELLS is as follows: +# +# GFDLgrid_NUM_CELLS typical cell size +# ------------------ ----------------- +# 192 50 km +# 384 25 km +# 768 13 km +# 1152 8.5 km +# 3072 3.2 km # # Note that these are only typical cell sizes. The actual cell size on # the global grid tiles varies somewhat as we move across a tile. @@ -835,10 +835,10 @@ GRID_GEN_METHOD="" # 1 (but still greater than 0) expands it. The remaining 5 tiles change # shape as necessary to maintain global coverage of the grid. # -# * The cell size on a given global tile depends on both GFDLgrid_RES and -# GFDLgrid_STRETCH_FAC (since changing GFDLgrid_RES changes the number -# of cells in the tile, and changing GFDLgrid_STRETCH_FAC modifies the -# shape and size of the tile). +# * The cell size on a given global tile depends on both GFDLgrid_NUM_CELLS +# and GFDLgrid_STRETCH_FAC (since changing GFDLgrid_NUM_CELLS changes +# the number of cells in the tile, and changing GFDLgrid_STRETCH_FAC +# modifies the shape and size of the tile). # # * The regional grid is embedded within tile 6 (i.e. it doesn't extend # beyond the boundary of tile 6). Its exact location within tile 6 is @@ -861,13 +861,13 @@ GRID_GEN_METHOD="" # # * GFDLgrid_REFINE_RATIO is the refinement ratio of the regional grid # (tile 7) with respect to the grid on its parent tile (tile 6), i.e. -# it is the number of grid cells along the boundary of the regional grid -# that abut one cell on tile 6. Thus, the cell size on the regional -# grid depends not only on GFDLgrid_RES and GFDLgrid_STRETCH_FAC (because -# the cell size on tile 6 depends on these two parameters) but also on -# GFDLgrid_REFINE_RATIO. Note that as on the tiles of the global grid, -# the cell size on the regional grid is not uniform but varies as we -# move across the grid. +# it is the number of grid cells along the boundary of the regional +# grid that abut one cell on tile 6. Thus, the cell size on the regional +# grid depends not only on GFDLgrid_NUM_CELLS and GFDLgrid_STRETCH_FAC +# (because the cell size on tile 6 depends on these two parameters) +# but also on GFDLgrid_REFINE_RATIO. Note that as on the tiles of the +# global grid, the cell size on the regional grid is not uniform but +# varies as we move across the grid. # # Definitions of parameters that need to be specified when GRID_GEN_METHOD # is set to "GFDLgrid": @@ -878,14 +878,9 @@ GRID_GEN_METHOD="" # GFDLgrid_LAT_T6_CTR: # Latitude of the center of tile 6 (in degrees). # -# GFDLgrid_RES: -# Number of points in each of the two horizontal directions (x and y) on -# each tile of the parent global grid. Note that the name of this parameter -# is really a misnomer because although it has the stirng "RES" (for -# "resolution") in its name, it specifies number of grid cells, not grid -# size (in say meters or kilometers). However, we keep this name in order -# to remain consistent with the usage of the word "resolution" in the -# global forecast model and other auxiliary codes. +# GFDLgrid_NUM_CELLS: +# Number of grid cells in each of the two horizontal directions (x and +# y) on each tile of the parent global grid. # # GFDLgrid_STRETCH_FAC: # Stretching factor used in the Schmidt transformation applied to the @@ -908,29 +903,30 @@ GRID_GEN_METHOD="" # GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: # j-index on tile 6 at which the regional grid (tile 7) ends. # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: -# Flag that determines the file naming convention to use for grid, orography, -# and surface climatology files (or, if using pregenerated files, the -# naming convention that was used to name these files). These files -# usually start with the string "C${RES}_", where RES is an integer. -# In the global forecast model, RES is the number of points in each of -# the two horizontal directions (x and y) on each tile of the global grid -# (defined here as GFDLgrid_RES). If this flag is set to "TRUE", RES will -# be set to GFDLgrid_RES just as in the global forecast model. If it is -# set to "FALSE", we calculate (in the grid generation task) an "equivalent -# global uniform cubed-sphere resolution" -- call it RES_EQUIV -- and -# then set RES equal to it. RES_EQUIV is the number of grid points in -# each of the x and y directions on each tile that a global UNIFORM (i.e. -# stretch factor of 1) cubed-sphere grid would have to have in order to -# have the same average grid size as the regional grid. This is a more -# useful indicator of the grid size because it takes into account the -# effects of GFDLgrid_RES, GFDLgrid_STRETCH_FAC, and GFDLgrid_REFINE_RATIO -# in determining the regional grid's typical grid size, whereas simply -# setting RES to GFDLgrid_RES doesn't take into account the effects of +# GFDLgrid_USE_NUM_CELLS_IN_FILENAMES: +# Flag that determines the file naming convention to use for grid, +# orography, and surface climatology files (or, if using pregenerated +# files, the naming convention that was used to name these files). +# These files usually start with the string "C${RES}_", where RES is an +# integer. In the global forecast model, RES is the number of grid +# cells in each of the two horizontal directions (x and y) on each tile +# of the global grid (defined here as GFDLgrid_NUM_CELLS). If this flag +# is set to "TRUE", RES will be set to GFDLgrid_NUM_CELLS just as in the +# global forecast model. If it is set to "FALSE", we calculate (in the +# grid generation task) an "equivalent global uniform cubed-sphere +# resolution" -- call it RES_EQUIV -- and then set RES equal to it. +# RES_EQUIV is the number of grid points in each of the x and y directions +# on each tile that a global UNIFORM (i.e. stretch factor of 1) cubed- +# sphere grid would have to have in order to have the same average grid +# size as the regional grid. This is a more useful indicator of the grid +# size because it takes into account the effects of GFDLgrid_NUM_CELLS, +# GFDLgrid_STRETCH_FAC, and GFDLgrid_REFINE_RATIO in determining the +# regional grid's typical grid size, whereas simply setting RES to +# GFDLgrid_NUM_CELLS doesn't take into account the effects of # GFDLgrid_STRETCH_FAC and GFDLgrid_REFINE_RATIO on the regional grid's -# resolution. Nevertheless, some users still prefer to use GFDLgrid_RES -# in the file names, so we allow for that here by setting this flag to -# "TRUE". +# resolution. Nevertheless, some users still prefer to use +# GFDLgrid_NUM_CELLS in the file names, so we allow for that here by +# setting this flag to "TRUE". # # Note that: # @@ -964,14 +960,14 @@ GRID_GEN_METHOD="" # GFDLgrid_LON_T6_CTR="" GFDLgrid_LAT_T6_CTR="" -GFDLgrid_RES="" +GFDLgrid_NUM_CELLS="" GFDLgrid_STRETCH_FAC="" GFDLgrid_REFINE_RATIO="" GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="" GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G="" GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G="" GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G="" -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES="" +GFDLgrid_USE_NUM_CELLS_IN_FILENAMES="" # #----------------------------------------------------------------------- # @@ -1651,7 +1647,7 @@ PPN_VX_ENSPOINT_PROB="1" # Walltimes. # WTIME_MAKE_GRID="00:20:00" -WTIME_MAKE_OROG="01:00:00" +WTIME_MAKE_OROG="00:20:00" WTIME_MAKE_SFC_CLIMO="00:20:00" WTIME_GET_EXTRN_ICS="00:45:00" WTIME_GET_EXTRN_LBCS="00:45:00" diff --git a/ush/link_fix.sh b/ush/link_fix.sh index 0bfd5c03d8..48bf0ca3f8 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -398,29 +398,9 @@ Please ensure that all files have the same resolution." #----------------------------------------------------------------------- # if [ "${file_group}" = "grid" ]; then - target="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" symlink="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" -# -# The surface climatology file generation code looks for a grid file -# having a name of the form "C${GFDLgrid_RES}_grid.tile7.halo4.nc" (i.e. -# the C-resolution used in the name of this file is the number of grid -# points per horizontal direction per tile, just like in the global model). -# Thus, if we are running the MAKE_SFC_CLIMO_TN task, if the grid is of -# GFDLgrid type, and if we are not using GFDLgrid_RES in filenames (i.e. -# we are using the equivalent global uniform grid resolution instead), -# then create a link whose name uses the GFDLgrid_RES that points to the -# link whose name uses the equivalent global uniform resolution. -# - if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] && \ - [ "${GRID_GEN_METHOD}" = "GFDLgrid" ] && \ - [ "${GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES}" = "FALSE" ]; then - target="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" - symlink="C${GFDLgrid_RES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" - create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" - fi - fi # #----------------------------------------------------------------------- diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh index 1794452d16..c4b6b038d7 100644 --- a/ush/machine/wcoss_dell_p3.sh +++ b/ush/machine/wcoss_dell_p3.sh @@ -83,3 +83,7 @@ MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" TEST_COMIN="${staged_data_dir}/COMGFS" TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" + +# Computational resources +WTIME_MAKE_OROG="01:00:00" + diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh index 71c3d12a7e..e0c89644df 100644 --- a/ush/set_predef_grid_params.sh +++ b/ush/set_predef_grid_params.sh @@ -82,13 +82,13 @@ function set_predef_grid_params() { "outvarname_gfdlgrid_lon_t6_ctr" \ "outvarname_gfdlgrid_lat_t6_ctr" \ "outvarname_gfdlgrid_stretch_fac" \ - "outvarname_gfdlgrid_res" \ + "outvarname_gfdlgrid_num_cells" \ "outvarname_gfdlgrid_refine_ratio" \ "outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g" \ "outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g" \ "outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g" \ "outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g" \ - "outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames" \ + "outvarname_gfdlgrid_use_num_cells_in_filenames" \ "outvarname_dt_atmos" \ "outvarname_layout_x" \ "outvarname_layout_y" \ @@ -131,13 +131,13 @@ function set_predef_grid_params() { __gfdlgrid_lon_t6_ctr__="" \ __gfdlgrid_lat_t6_ctr__="" \ __gfdlgrid_stretch_fac__="" \ - __gfdlgrid_res__="" \ + __gfdlgrid_num_cells__="" \ __gfdlgrid_refine_ratio__="" \ __gfdlgrid_istart_of_rgnl_dom_on_t6g__="" \ __gfdlgrid_iend_of_rgnl_dom_on_t6g__="" \ __gfdlgrid_jstart_of_rgnl_dom_on_t6g__="" \ __gfdlgrid_jend_of_rgnl_dom_on_t6g__="" \ - __gfdlgrid_use_gfdlgrid_res_in_filenames__="" \ + __gfdlgrid_use_num_cells_in_filenames__="" \ __dt_atmos__="" \ __layout_x__="" \ __layout_y__="" \ @@ -694,22 +694,22 @@ function set_predef_grid_params() { # __gfdlgrid_lon_t6_ctr__="-160.8" # __gfdlgrid_lat_t6_ctr__="63.0" # __gfdlgrid_stretch_fac__="1.161" -# __gfdlgrid_res__="768" +# __gfdlgrid_num_cells__="768" # __gfdlgrid_refine_ratio__="4" # # num_margin_cells_T6_left="204" # __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) # # num_margin_cells_T6_right="204" -# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) +# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) # # num_margin_cells_T6_bottom="249" # __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) # # num_margin_cells_T6_top="249" -# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) +# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) # -# __gfdlgrid_use_gfdlgrid_res_in_filenames__="FALSE" +# __gfdlgrid_use_num_cells_in_filenames__="FALSE" # # __dt_atmos__="${dt_atmos:-18}" # @@ -845,22 +845,22 @@ function set_predef_grid_params() { __gfdlgrid_lon_t6_ctr__="-97.5" __gfdlgrid_lat_t6_ctr__="38.5" __gfdlgrid_stretch_fac__="1.4" - __gfdlgrid_res__="96" + __gfdlgrid_num_cells__="96" __gfdlgrid_refine_ratio__="3" num_margin_cells_T6_left="12" __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) num_margin_cells_T6_right="12" - __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) + __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) num_margin_cells_T6_bottom="16" __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) num_margin_cells_T6_top="16" - __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) + __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) - __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" + __gfdlgrid_use_num_cells_in_filenames__="TRUE" __dt_atmos__="${dt_atmos:-225}" @@ -900,22 +900,22 @@ function set_predef_grid_params() { __gfdlgrid_lon_t6_ctr__="-97.5" __gfdlgrid_lat_t6_ctr__="38.5" __gfdlgrid_stretch_fac__="1.5" - __gfdlgrid_res__="768" + __gfdlgrid_num_cells__="768" __gfdlgrid_refine_ratio__="3" num_margin_cells_T6_left="69" __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) num_margin_cells_T6_right="69" - __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) + __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) num_margin_cells_T6_bottom="164" __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) num_margin_cells_T6_top="164" - __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) + __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) - __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" + __gfdlgrid_use_num_cells_in_filenames__="TRUE" __dt_atmos__="${dt_atmos:-18}" @@ -985,22 +985,22 @@ function set_predef_grid_params() { # __gfdlgrid_lon_t6_ctr__="-153.0" # __gfdlgrid_lat_t6_ctr__="61.0" # __gfdlgrid_stretch_fac__="1.0" # ??? -# __gfdlgrid_res__="768" +# __gfdlgrid_num_cells__="768" # __gfdlgrid_refine_ratio__="3" # ??? # # num_margin_cells_T6_left="61" # __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) # # num_margin_cells_T6_right="67" -# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_right )) +# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) # # num_margin_cells_T6_bottom="165" # __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) # # num_margin_cells_T6_top="171" -# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_res__ - num_margin_cells_T6_top )) +# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) # -# __gfdlgrid_use_gfdlgrid_res_in_filenames__="TRUE" +# __gfdlgrid_use_num_cells_in_filenames__="TRUE" # # __dt_atmos__="${dt_atmos:-18}" # @@ -1641,8 +1641,8 @@ function set_predef_grid_params() { printf -v ${outvarname_gfdlgrid_stretch_fac} "%s" "${__gfdlgrid_stretch_fac__}" fi - if [ ! -z "${outvarname_gfdlgrid_res}" ]; then - printf -v ${outvarname_gfdlgrid_res} "%s" "${__gfdlgrid_res__}" + if [ ! -z "${outvarname_gfdlgrid_num_cells}" ]; then + printf -v ${outvarname_gfdlgrid_num_cells} "%s" "${__gfdlgrid_num_cells__}" fi if [ ! -z "${outvarname_gfdlgrid_refine_ratio}" ]; then @@ -1665,8 +1665,8 @@ function set_predef_grid_params() { printf -v ${outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_jend_of_rgnl_dom_on_t6g__}" fi - if [ ! -z "${outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames}" ]; then - printf -v ${outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames} "%s" "${__gfdlgrid_use_gfdlgrid_res_in_filenames__}" + if [ ! -z "${outvarname_gfdlgrid_use_num_cells_in_filenames}" ]; then + printf -v ${outvarname_gfdlgrid_use_num_cells_in_filenames} "%s" "${__gfdlgrid_use_num_cells_in_filenames__}" fi if [ ! -z "${outvarname_dt_atmos}" ]; then diff --git a/ush/setup.sh b/ush/setup.sh index 108080db23..8ba9eab852 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -951,13 +951,13 @@ if [ ! -z "${PREDEF_GRID_NAME}" ]; then outvarname_gfdlgrid_lon_t6_ctr="GFDLgrid_LON_T6_CTR" \ outvarname_gfdlgrid_lat_t6_ctr="GFDLgrid_LAT_T6_CTR" \ outvarname_gfdlgrid_stretch_fac="GFDLgrid_STRETCH_FAC" \ - outvarname_gfdlgrid_res="GFDLgrid_RES" \ + outvarname_gfdlgrid_num_cells="GFDLgrid_NUM_CELLS" \ outvarname_gfdlgrid_refine_ratio="GFDLgrid_REFINE_RATIO" \ outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G" \ outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G" \ outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g="GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G" \ outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g="GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G" \ - outvarname_gfdlgrid_use_gfdlgrid_res_in_filenames="GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES" \ + outvarname_gfdlgrid_use_num_cells_in_filenames="GFDLgrid_USE_NUM_CELLS_IN_FILENAMES" \ outvarname_dt_atmos="DT_ATMOS" \ outvarname_layout_x="LAYOUT_X" \ outvarname_layout_y="LAYOUT_Y" \ @@ -997,17 +997,17 @@ check_var_valid_value \ # #----------------------------------------------------------------------- # -# For a "GFDLgrid" type of grid, make sure GFDLgrid_RES is set to a valid -# value. +# For a "GFDLgrid" type of grid, make sure GFDLgrid_NUM_CELLS is set to +# a valid value. # #----------------------------------------------------------------------- # if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then err_msg="\ The number of grid cells per tile in each horizontal direction specified -in GFDLgrid_RES is not supported: - GFDLgrid_RES = \"${GFDLgrid_RES}\"" - check_var_valid_value "GFDLgrid_RES" "valid_vals_GFDLgrid_RES" "${err_msg}" +in GFDLgrid_NUM_CELLS is not supported: + GFDLgrid_NUM_CELLS = \"${GFDLgrid_NUM_CELLS}\"" + check_var_valid_value "GFDLgrid_NUM_CELLS" "valid_vals_GFDLgrid_NUM_CELLS" "${err_msg}" fi # #----------------------------------------------------------------------- @@ -1903,7 +1903,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then set_gridparams_GFDLgrid \ lon_of_t6_ctr="${GFDLgrid_LON_T6_CTR}" \ lat_of_t6_ctr="${GFDLgrid_LAT_T6_CTR}" \ - res_of_t6g="${GFDLgrid_RES}" \ + res_of_t6g="${GFDLgrid_NUM_CELLS}" \ stretch_factor="${GFDLgrid_STRETCH_FAC}" \ refine_ratio_t6g_to_t7g="${GFDLgrid_REFINE_RATIO}" \ istart_of_t7_on_t6g="${GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G}" \ diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index c3b0fe8f9d..cd765cb2b3 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -45,7 +45,7 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_RRFS_v1alpha" \ "FV3_HRRR" \ ) -valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") +valid_vals_GFDLgrid_NUM_CELLS=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2" "netcdf") From 64013c88231fe938d648ea975ade52cd04c5ecc0 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:52:18 -0600 Subject: [PATCH 188/203] Remove un-readable path to systemdir on Hera. (#806) This is a bug fix for running the WE2E tests on Hera. The non-existence of this path is causing failures after the path has become unreadable. --- ush/machine/hera.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh index f8c9324a97..b34610d44e 100644 --- a/ush/machine/hera.sh +++ b/ush/machine/hera.sh @@ -3,6 +3,7 @@ function file_location() { # Return the default location of external model files on disk + # Hera does not currently have any files staged on disk. local external_file_fmt external_model location @@ -10,13 +11,6 @@ function file_location() { external_file_fmt=${2} location="" - case ${external_model} in - - "FV3GFS") - location='/scratch1/NCEPDEV/rstprod/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' - ;; - - esac echo ${location:-} } From 761b6c383b4082587870d00375a6c2245c557ab1 Mon Sep 17 00:00:00 2001 From: EdwardSnyder-NOAA <96196752+EdwardSnyder-NOAA@users.noreply.github.com> Date: Fri, 1 Jul 2022 12:04:56 -0500 Subject: [PATCH 189/203] [develop] Add MET/METplus to Jet (#799) * add metplus paths * add run_vx.local for jet Co-authored-by: Edward Snyder --- modulefiles/tasks/jet/run_vx.local | 5 +++++ ush/machine/jet.sh | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 modulefiles/tasks/jet/run_vx.local diff --git a/modulefiles/tasks/jet/run_vx.local b/modulefiles/tasks/jet/run_vx.local new file mode 100644 index 0000000000..9be1ed3c79 --- /dev/null +++ b/modulefiles/tasks/jet/run_vx.local @@ -0,0 +1,5 @@ +#%Module + +module use -a /contrib/anaconda/modulefiles +module load intel/18.0.5.274 +module load anaconda/5.3.1 diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh index f383090ec5..ced5345de4 100644 --- a/ush/machine/jet.sh +++ b/ush/machine/jet.sh @@ -81,6 +81,14 @@ RUN_CMD_UTILS="srun" RUN_CMD_FCST="srun" RUN_CMD_POST="srun" +# MET/METplus-Related Paths +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.1.1"} +METPLUS_PATH=${METPLUS_PATH:-"/contrib/met/METplus/METplus-4.1.1"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} + # Test Data Locations TEST_COMIN="${staged_data_dir}/COMGFS" TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" From 553d01af4aee45a05c059fb10fbdb3a99fe479fa Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 12 Jul 2022 05:42:26 -0400 Subject: [PATCH 190/203] Remove RRFS_v1alpha suite (#809) * Remove RRFS_v1alpha suite --- scripts/exregional_make_ics.sh | 1 - scripts/exregional_make_lbcs.sh | 1 - tests/WE2E/machine_suites/hera.txt | 1 - tests/WE2E/machine_suites/jet.txt | 1 - ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 27 -- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 27 -- ...km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh | 28 -- ...s_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh} | 4 +- ush/templates/FV3.input.yml | 33 +- ush/templates/diag_table.FV3_RRFS_v1alpha | 349 ------------------ ush/templates/field_table.FV3_RRFS_v1alpha | 65 ---- ush/valid_param_vals.sh | 1 - ush/valid_param_vals.yaml | 1 - 13 files changed, 16 insertions(+), 523 deletions(-) delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh delete mode 100644 tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh rename tests/WE2E/test_configs/grids_extrn_mdls_suites_community/{config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh => config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh} (97%) delete mode 100755 ush/templates/diag_table.FV3_RRFS_v1alpha delete mode 100644 ush/templates/field_table.FV3_RRFS_v1alpha diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 7af832c4ce..b7a8dbe05a 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -142,7 +142,6 @@ case "${CCPP_PHYS_SUITE}" in varmap_file="GFSphys_var_map.txt" ;; # - "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ "FV3_GFS_v15_thompson_mynn_lam3km" | \ "FV3_HRRR" ) diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index a98ab354df..6dc3ba369a 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -140,7 +140,6 @@ case "${CCPP_PHYS_SUITE}" in varmap_file="GFSphys_var_map.txt" ;; # - "FV3_RRFS_v1alpha" | \ "FV3_RRFS_v1beta" | \ "FV3_GFS_v15_thompson_mynn_lam3km" | \ "FV3_HRRR" ) diff --git a/tests/WE2E/machine_suites/hera.txt b/tests/WE2E/machine_suites/hera.txt index a3f7f28509..0887e6c58e 100644 --- a/tests/WE2E/machine_suites/hera.txt +++ b/tests/WE2E/machine_suites/hera.txt @@ -5,6 +5,5 @@ grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/machine_suites/jet.txt b/tests/WE2E/machine_suites/jet.txt index a3f7f28509..0887e6c58e 100644 --- a/tests/WE2E/machine_suites/jet.txt +++ b/tests/WE2E/machine_suites/jet.txt @@ -5,6 +5,5 @@ grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh deleted file mode 100644 index f44afffcda..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ /dev/null @@ -1,27 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUScompact_13km grid using the RRFS_v1alpha -# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUScompact_13km" -CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" -EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) -EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh deleted file mode 100644 index bf2e2f15e5..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ /dev/null @@ -1,27 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1alpha -# physics suite with ICs derived from the HRRR and LBCs derived from the RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUScompact_25km" -CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" -EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) -EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) - -DATE_FIRST_CYCL="20200810" -DATE_LAST_CYCL="20200810" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="24" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh deleted file mode 100644 index cc92aecaaa..0000000000 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ /dev/null @@ -1,28 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_CONUScompact_3km grid using the -# RRFS_v1alpha physics suite with ICs derived from the HRRR and LBCs -# derived from the RAP. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUScompact_3km" -CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" - -EXTRN_MDL_NAME_ICS="HRRR" -EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" -EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) -EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) - -DATE_FIRST_CYCL="20200801" -DATE_LAST_CYCL="20200801" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh similarity index 97% rename from tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh rename to tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh index f743ebb40e..cd0a933e6c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1beta.sh @@ -3,7 +3,7 @@ # ------------------------ # # This test is to ensure that the workflow running in community mode -# completes successfully on the RRFS_NA_3km grid using the RRFS_v1alpha +# completes successfully on the RRFS_NA_3km grid using the RRFS_v1beta # physics suite with ICs and LBCs derived from the FV3GFS. # # Note that this test also sets various resource parameters for several @@ -15,7 +15,7 @@ RUN_ENVIR="community" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_NA_3km" -CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" +CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 7b5dab6d77..0e9276de25 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -12,10 +12,10 @@ # and provide this file and the desired section via the -c option. -FV3_RRFS_v1alpha: - gfs_physics_nml: &RRFS_v1alpha_phys +FV3_RRFS_v1beta: + gfs_physics_nml: &RRFS_v1beta_phys do_deep: False - do_mynnsfclay: False + do_mynnsfclay: True imfdeepcnv: -1 imfshalcnv: -1 iopt_alb: 2 @@ -34,11 +34,6 @@ FV3_RRFS_v1alpha: lsm: 2 lsoil_lsm: 4 -FV3_RRFS_v1beta: - gfs_physics_nml: - <<: *RRFS_v1alpha_phys - do_mynnsfclay: True - FV3_HRRR: fv_core_nml: hord_dp: -5 @@ -52,7 +47,7 @@ FV3_HRRR: nord_tr: 2 nrows_blend: 10 gfs_physics_nml: - <<: *RRFS_v1alpha_phys + <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True do_sfcperts: !!python/none @@ -282,19 +277,19 @@ FV3_GFS_v15_thompson_mynn_lam3km: atmos_model_nml: avg_max_length: 3600.0 fv_core_nml: - agrid_vel_rst: true + agrid_vel_rst: True full_zs_filter: !!python/none n_sponge: 9 npz_type: '' - rf_fast: false + rf_fast: False sg_cutoff: 10000.0 vtdm4: 0.02 gfs_physics_nml: avg_max_length: 3600.0 cdmbgwd: [0.88, 0.04] - do_deep: false - do_mynnsfclay: true - do_ugwp: false + do_deep: False + do_mynnsfclay: True + do_ugwp: False fhswr: 900.0 fhlwr: 900.0 iaer: 1011 @@ -316,11 +311,11 @@ FV3_GFS_v15_thompson_mynn_lam3km: iopt_tbot: 2 iopt_trs: 2 iovr: 3 - ldiag_ugwp: false - lgfdlmprad: false + ldiag_ugwp: False + lgfdlmprad: False lsm: 1 lsoil_lsm: !!python/none - ltaerosol: false + ltaerosol: False xkzminv: 0.3 xkzm_m: 1.0 xkzm_h: 1.0 @@ -330,7 +325,7 @@ FV3_GFS_v16: launch_level: 27 fv_core_nml: <<: *gfs_v15_fv_core - agrid_vel_rst: false + agrid_vel_rst: False d2_bg_k1: 0.2 d2_bg_k2: 0.0 fv_sg_adj: 450 @@ -339,7 +334,7 @@ FV3_GFS_v16: hord_tm: 5 hord_vt: 5 k_split: 6 - make_nh: false + make_nh: False n_split: 6 n_sponge: 10 na_init: 0 diff --git a/ush/templates/diag_table.FV3_RRFS_v1alpha b/ush/templates/diag_table.FV3_RRFS_v1alpha deleted file mode 100755 index b97d67f61a..0000000000 --- a/ush/templates/diag_table.FV3_RRFS_v1alpha +++ /dev/null @@ -1,349 +0,0 @@ -{{ starttime.strftime("%Y%m%d.%H") }}Z.{{ cres }}.32bit.non-hydro.regional -{{ starttime.strftime("%Y %m %d %H %M %S") }} - -"grid_spec", -1, "months", 1, "days", "time" -"atmos_static", -1, "hours", 1, "hours", "time" -#"atmos_4xdaily", 1, "hours", 1, "days", "time" -"fv3_history", 1, "years", 1, "hours", "time" -"fv3_history2d", 1, "years", 1, "hours", "time" - -# -#======================= -# ATMOSPHERE DIAGNOSTICS -#======================= -### -# grid_spec -### - "dynamics", "grid_lon", "grid_lon", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lat", "grid_lat", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_lont", "grid_lont", "grid_spec", "all", .false., "none", 2, - "dynamics", "grid_latt", "grid_latt", "grid_spec", "all", .false., "none", 2, - "dynamics", "area", "area", "grid_spec", "all", .false., "none", 2, -### -# 4x daily output -### -# "dynamics", "slp", "slp", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort850", "vort850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vort200", "vort200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "us", "us", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u1000", "u1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u850", "u850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u700", "u700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u500", "u500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u200", "u200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u100", "u100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u50", "u50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "u10", "u10", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "vs", "vs", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v1000", "v1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v850", "v850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v700", "v700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v500", "v500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v200", "v200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v100", "v100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v50", "v50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "v10", "v10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "tm", "tm", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t1000", "t1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t850", "t850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t700", "t700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t500", "t500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t200", "t200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t100", "t100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t50", "t50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "t10", "t10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "z1000", "z1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z850", "z850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z700", "z700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z500", "z500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z200", "z200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z100", "z100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z50", "z50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "z10", "z10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "w1000", "w1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w850", "w850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w700", "w700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w500", "w500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "w200", "w200", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "q1000", "q1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q850", "q850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q700", "q700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q500", "q500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q200", "q200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q100", "q100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q50", "q50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "q10", "q10", "atmos_4xdaily", "all", .false., "none", 2 -#### -# "dynamics", "rh1000", "rh1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh850", "rh850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh700", "rh700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh500", "rh500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "rh200", "rh200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg1000", "omg1000", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg850", "omg850", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg700", "omg700", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg500", "omg500", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg200", "omg200", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg100", "omg100", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg50", "omg50", "atmos_4xdaily", "all", .false., "none", 2 -# "dynamics", "omg10", "omg10", "atmos_4xdaily", "all", .false., "none", 2 -### -# gfs static data -### - "dynamics", "pk", "pk", "atmos_static", "all", .false., "none", 2 - "dynamics", "bk", "bk", "atmos_static", "all", .false., "none", 2 - "dynamics", "hyam", "hyam", "atmos_static", "all", .false., "none", 2 - "dynamics", "hybm", "hybm", "atmos_static", "all", .false., "none", 2 - "dynamics", "zsurf", "zsurf", "atmos_static", "all", .false., "none", 2 -### -# FV3 variabls needed for NGGPS evaluation -### -"gfs_dyn", "ucomp", "ugrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "vcomp", "vgrd", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sphum", "spfh", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "temp", "tmp", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "liq_wat", "clwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "o3mr", "o3mr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delp", "dpres", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "delz", "delz", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "w", "dzdt", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ice_wat", "icmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "rainwat", "rwmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "snowwat", "snmr", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "graupel", "grle", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "ps", "pressfc", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "hs", "hgtsfc", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "refl_10cm" "refl_10cm" "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cldfra", "cldfra", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "ice_nc", "nicp", "fv3_history", "all", .false., "none", 2 -#"gfs_dyn", "rain_nc", "ntrnc", "fv3_history", "all", .false., "none", 2 - -"gfs_dyn", "wmaxup", "upvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "wmaxdn", "dnvvelmax", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax03", "uhmax03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmax25", "uhmax25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin03", "uhmin03", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "uhmin25", "uhmin25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort01", "maxvort01", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvort02", "maxvort02", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "maxvorthy1", "maxvorthy1", "fv3_history", "all", .false., "none", 2 - -"gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cnvprcpb_ave", "cpratb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcp_ave", "prate_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "totprcpb_ave", "prateb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRF", "dlwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DLWRFI", "dlwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRF", "ulwrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFI", "ulwrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRF", "dswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFI", "dswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRF", "uswrf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFI", "uswrf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "DSWRFtoa", "dswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "USWRFtoa", "uswrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ULWRFtoa", "ulwrf_avetoa","fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gflux_ave", "gflux_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hpbl", "hpbl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl_ave", "lhtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl_ave", "shtfl_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pwat", "pwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "soilm", "soilm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_aveclm", "tcdc_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avebndcl", "tcdc_avebndcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avehcl", "tcdc_avehcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avelcl", "tcdc_avelcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDC_avemcl", "tcdc_avemcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TCDCcnvcl", "tcdccnvcl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclt", "prescnvclt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PREScnvclb", "prescnvclb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehct", "pres_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avehcb", "pres_avehcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avehct", "tmp_avehct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemct", "pres_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avemcb", "pres_avemcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avemct", "tmp_avemct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelct", "pres_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "PRES_avelcb", "pres_avelcb", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "TEMP_avelct", "tmp_avelct", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u-gwd_ave", "u-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v-gwd_ave", "v-gwd_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dusfc", "uflx_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "dvsfc", "vflx_ave", "fv3_history2d", "all", .false., "none", 2 -#"gfs_phys", "cnvw", "cnvcldwat", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "psurf", "pressfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "u10m", "ugrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "v10m", "vgrd10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "crain", "crain", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tprcp", "tprcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hgtsfc", "orog", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "weasd", "weasd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "f10m", "f10m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "q2m", "spfh2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "t2m", "tmp2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tsfc", "tmpsfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vtype", "vtype", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "stype", "sotyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slmsksfc", "land", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "vfracsfc", "veg", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zorlsfc", "sfcr", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "uustar", "fricv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt1", "soilt1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt2", "soilt2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt3", "soilt3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilt4", "soilt4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw1", "soilw1" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw2", "soilw2" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw3", "soilw3" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "soilw4", "soilw4" "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_1", "soill1", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_2", "soill2", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_3", "soill3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slc_4", "soill4", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "slope", "sltyp", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnsf", "alnsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alnwf", "alnwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvsf", "alvsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "alvwf", "alvwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "canopy", "cnwat", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facsf", "facsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "facwf", "facwf", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffhh", "ffhh", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "ffmm", "ffmm", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "fice", "icec", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "hice", "icetk", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snoalb", "snoalb", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmax", "shdmax", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "shdmin", "shdmin", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "snowd", "snod", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tg3", "tg3", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tisfc", "tisfc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "tref", "tref", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "z_c", "zc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_0", "c0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "c_d", "cd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_0", "w0", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "w_d", "wd", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xt", "xt", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xz", "xz", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "dt_cool", "dtcool", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xs", "xs", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xu", "xu", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xv", "xv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xtts", "xtts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "xzts", "xzts", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "d_conv", "dconv", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "qrain", "qrain", "fv3_history2d", "all", .false., "none", 2 - -"gfs_phys", "acond", "acond", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cduvb_ave", "cduvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cpofp", "cpofp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "duvb_ave", "duvb_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdlf_ave", "csdlf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_ave", "csusf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csusf_avetoa", "csusftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csdsf_ave", "csdsf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_ave", "csulf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "csulf_avetoa", "csulftoa", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "cwork_ave", "cwork_aveclm", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evbs_ave", "evbs_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "evcw_ave", "evcw_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "fldcp", "fldcp", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "hgt_hyblev1", "hgt_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfh_hyblev1", "spfh_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ugrd_hyblev1", "ugrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vgrd_hyblev1", "vgrd_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmp_hyblev1", "tmp_hyblev1", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "gfluxi", "gflux", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "lhtfl", "lhtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "shtfl", "shtfl", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr", "pevpr", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "pevpr_ave", "pevpr_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sbsno_ave", "sbsno_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sfexc", "sfexc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snohf", "snohf", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "snowc_ave", "snowc_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmax2m", "spfhmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "spfhmin2m", "spfhmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmax2m", "tmax_max2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "tmpmin2m", "tmin_min2m", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "ssrun_acc", "ssrun_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "sunsd_acc", "sunsd_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "watr_acc", "watr_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "wilt", "wilt", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vbdsf_ave", "vbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "vddsf_ave", "vddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nbdsf_ave", "nbdsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "nddsf_ave", "nddsf_ave", "fv3_history2d", "all", .false., "none", 2 -"gfs_phys", "trans_ave", "trans_ave", "fv3_history2d", "all", .false., "none", 2 -# Aerosols (CCN, IN) from Thompson microphysics -"gfs_phys", "nwfa", "nwfa", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "nifa", "nifa", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "nwfa2d", "nwfa2d", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "nifa2d", "nifa2d", "fv3_history2d", "all", .false., "none", 2 -# Cloud effective radii from Thompson and WSM6 microphysics -"gfs_phys", "cleffr", "cleffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cieffr", "cieffr", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "cseffr", "cseffr", "fv3_history", "all", .false., "none", 2 -# Prognostic/diagnostic variables from MYNN -"gfs_phys", "QC_BL", "qc_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "CLDFRA_BL", "cldfra_bl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "EL_PBL", "el_pbl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "QKE", "qke", "fv3_history", "all", .false., "none", 2 -"gfs_sfc", "maxmf", "maxmf", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "nupdraft", "nupdrafts", "fv3_history2d", "all", .false., "none", 2 -#"gfs_sfc", "ktop_shallow", "ktop_shallow", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "zol", "zol", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flhc", "flhc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "flqc", "flqc", "fv3_history2d", "all", .false., "none", 2 -# Prognostic/diagnostic variables from RUC LSM -"gfs_sfc", "snowfall_acc", "snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -"gfs_sfc", "swe_snowfall_acc", "swe_snowfall_acc", "fv3_history2d", "all", .false., "none", 2 -# Stochastic physics -"gfs_phys", "sppt_wts", "sppt_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebu_wts", "skebu_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "skebv_wts", "skebv_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "shum_wts", "shum_wts", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "spp_wts_pbl", "spp_wts_pbl", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "spp_wts_mp", "spp_wts_mp", "fv3_history", "all", .false., "none", 2 -"gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 - -#============================================================================================= -# -#====> This file can be used with diag_manager/v2.0a (or higher) <==== -# -# -# FORMATS FOR FILE ENTRIES (not all input values are used) -# ------------------------ -# -#"file_name", output_freq, "output_units", format, "time_units", "long_name", -# -# -#output_freq: > 0 output frequency in "output_units" -# = 0 output frequency every time step -# =-1 output frequency at end of run -# -#output_units = units used for output frequency -# (years, months, days, minutes, hours, seconds) -# -#time_units = units used to label the time axis -# (days, minutes, hours, seconds) -# -# -# FORMAT FOR FIELD ENTRIES (not all input values are used) -# ------------------------ -# -#"module_name", "field_name", "output_name", "file_name" "time_sampling", time_avg, "other_opts", packing -# -#time_avg = .true. or .false. -# -#packing = 1 double precision -# = 2 float -# = 4 packed 16-bit integers -# = 8 packed 1-byte (not tested?) diff --git a/ush/templates/field_table.FV3_RRFS_v1alpha b/ush/templates/field_table.FV3_RRFS_v1alpha deleted file mode 100644 index fe96567e5f..0000000000 --- a/ush/templates/field_table.FV3_RRFS_v1alpha +++ /dev/null @@ -1,65 +0,0 @@ -# added by FRE: sphum must be present in atmos -# specific humidity for moist runs - "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water mixing ratio - "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic ice water mixing ratio - "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic rain water mixing ratio - "TRACER", "atmos_mod", "rainwat" - "longname", "rain water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic snow water mixing ratio - "TRACER", "atmos_mod", "snowwat" - "longname", "snow water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic graupel mixing ratio - "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud water number concentration - "TRACER", "atmos_mod", "water_nc" - "longname", "cloud liquid water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic cloud ice number concentration - "TRACER", "atmos_mod", "ice_nc" - "longname", "cloud ice water number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic rain number concentration - "TRACER", "atmos_mod", "rain_nc" - "longname", "rain number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic ozone mixing ratio tracer - "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# water- and ice-friendly aerosols (Thompson) - "TRACER", "atmos_mod", "liq_aero" - "longname", "water-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / - "TRACER", "atmos_mod", "ice_aero" - "longname", "ice-friendly aerosol number concentration" - "units", "/kg" - "profile_type", "fixed", "surface_value=0.0" / -# prognostic subgrid scale turbulent kinetic energy - "TRACER", "atmos_mod", "sgs_tke" - "longname", "subgrid scale turbulent kinetic energy" - "units", "m2/s2" - "profile_type", "fixed", "surface_value=0.0" / diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index cd765cb2b3..2755e5709f 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -42,7 +42,6 @@ valid_vals_CCPP_PHYS_SUITE=( \ "FV3_GFS_v15_thompson_mynn_lam3km" \ "FV3_GFS_v16" \ "FV3_RRFS_v1beta" \ -"FV3_RRFS_v1alpha" \ "FV3_HRRR" \ ) valid_vals_GFDLgrid_NUM_CELLS=("48" "96" "192" "384" "768" "1152" "3072") diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index efadce6088..4c701f777c 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -40,7 +40,6 @@ valid_vals_CCPP_PHYS_SUITE: [ "FV3_GFS_v15_thompson_mynn_lam3km", "FV3_GFS_v16", "FV3_RRFS_v1beta", -"FV3_RRFS_v1alpha", "FV3_HRRR" ] valid_vals_GFDLgrid_RES: [48, 96, 192, 384, 768, 1152, 3072] From e7a22403a66419f2866af384ec4af484255c7357 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Mon, 18 Jul 2022 11:10:30 -0600 Subject: [PATCH 191/203] [develop]: Remove shell workflow. (#764) * Bug fix with FIELD_TABLE_FN * Modify crontab management, use config_defaults.sh. * Add status badge. * Update cheyenne crontab management. * source lmod-setup * Add main to set_predef_grid * Bug fix in predef_grid * Don't import dead params. * Fix bug in resetting VERBOSE * Minor fix in INI config. * Construct var_defns components from dictionary. * Allow also lower case variables to be exported. * Updates to python workflow due to PR #776 * Use python versions of link_fix and set_FV3_sfc in job script. * Use python versions of create_diag/model. * Some fixes addressing Christina's suggestions. * Delete shell workflow * Append pid to temp files. * Update scripts to work with the latest hashes of UFS_UTILS and UPP (#775) * update input namelist of chgres_cube * update diag_table templates * update scripts * back to original * specify miniconda version on Jet * Remove -S option from link_fix call. * Fixes due to merge * Cosmoetic changes. Co-authored-by: Chan-Hoo.Jeon-NOAA <60152248+chan-hoo@users.noreply.github.com> --- README.md | 2 + scripts/exregional_make_grid.sh | 12 +- scripts/exregional_make_orog.sh | 14 +- scripts/exregional_make_sfc_climo.sh | 14 +- scripts/exregional_run_fcst.sh | 32 +- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 152 +- tests/WE2E/run_WE2E_tests.sh | 2 +- tests/WE2E/setup_WE2E_tests.sh | 1 + ush/calculate_cost.py | 106 + ush/check_ruc_lsm.sh | 120 - ush/config_defaults.sh | 11 +- ush/config_defaults.yaml | 2017 ------------ ush/constants.py | 27 - ush/constants.sh | 6 +- ush/create_diag_table_file.py | 29 +- ush/create_diag_table_file.sh | 117 - ush/create_model_configure_file.py | 59 +- ush/create_model_configure_file.sh | 257 -- ush/generate_FV3LAM_wflow.py | 74 +- ush/generate_FV3LAM_wflow.sh | 1213 ------- ush/get_crontab_contents.py | 126 +- ush/get_crontab_contents.sh | 74 - ush/launch_FV3LAM_wflow.sh | 34 +- ush/link_fix.py | 46 +- ush/link_fix.sh | 460 --- ush/predef_grid_params.yaml | 43 +- ush/python_utils/__init__.py | 47 +- ush/python_utils/config_parser.py | 23 +- ush/python_utils/environment.py | 25 +- ush/python_utils/misc.py | 18 + ush/set_FV3nml_ens_stoch_seeds.py | 39 +- ush/set_FV3nml_ens_stoch_seeds.sh | 187 -- ush/set_FV3nml_sfc_climo_filenames.py | 23 +- ush/set_FV3nml_sfc_climo_filenames.sh | 231 -- ush/set_cycle_dates.sh | 145 - ush/set_extrn_mdl_params.sh | 35 - ush/set_gridparams_ESGgrid.py | 33 +- ush/set_gridparams_ESGgrid.sh | 215 -- ush/set_gridparams_GFDLgrid.py | 2 - ush/set_gridparams_GFDLgrid.sh | 571 ---- ush/set_ozone_param.sh | 241 -- ush/set_predef_grid_params.py | 23 +- ush/set_predef_grid_params.sh | 1765 ---------- ush/set_thompson_mp_fix_files.sh | 192 -- ush/setup.py | 364 +-- ush/setup.sh | 2853 ----------------- ush/valid_param_vals.sh | 59 - ush/valid_param_vals.yaml | 2 +- 48 files changed, 714 insertions(+), 11427 deletions(-) create mode 100755 ush/calculate_cost.py delete mode 100644 ush/check_ruc_lsm.sh delete mode 100644 ush/config_defaults.yaml delete mode 100644 ush/constants.py delete mode 100644 ush/create_diag_table_file.sh delete mode 100644 ush/create_model_configure_file.sh delete mode 100755 ush/generate_FV3LAM_wflow.sh delete mode 100644 ush/get_crontab_contents.sh mode change 100644 => 100755 ush/link_fix.py delete mode 100755 ush/link_fix.sh delete mode 100644 ush/set_FV3nml_ens_stoch_seeds.sh delete mode 100644 ush/set_FV3nml_sfc_climo_filenames.sh delete mode 100644 ush/set_cycle_dates.sh delete mode 100644 ush/set_extrn_mdl_params.sh delete mode 100644 ush/set_gridparams_ESGgrid.sh delete mode 100644 ush/set_gridparams_GFDLgrid.sh delete mode 100644 ush/set_ozone_param.sh delete mode 100644 ush/set_predef_grid_params.sh delete mode 100644 ush/set_thompson_mp_fix_files.sh delete mode 100755 ush/setup.sh delete mode 100644 ush/valid_param_vals.sh diff --git a/README.md b/README.md index de61c44209..e0b3e9bd52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Python unittests](https://github.com/ufs-community/regional_workflow/actions/workflows/python_unittests.yaml/badge.svg)](https://github.com/ufs-community/regional_workflow/actions/workflows/python_unittests.yaml) + # Regional Workflow **The regional workflow in this repository can no longer be run in a stand-alone configuration. To clone the end-to-end system, build the code, and run the workflow, see:** diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index de27902e32..929be84dc5 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -17,8 +17,6 @@ #----------------------------------------------------------------------- # . $USHDIR/make_grid_mosaic_file.sh -. $USHDIR/link_fix.sh -. $USHDIR/set_FV3nml_sfc_climo_filenames.sh # #----------------------------------------------------------------------- # @@ -589,9 +587,9 @@ halo failed." # #----------------------------------------------------------------------- # -link_fix \ - verbose="$VERBOSE" \ - file_group="grid" || \ +python3 $USHDIR/link_fix.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --file-group "grid" || \ print_err_msg_exit "\ Call to function to create symlinks to the various grid and mosaic files failed." @@ -607,7 +605,9 @@ failed." # #----------------------------------------------------------------------- # -set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ +python3 $USHDIR/set_FV3nml_sfc_climo_filenames.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist file failed." # diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 58e453a97a..3904924433 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -12,14 +12,6 @@ # #----------------------------------------------------------------------- # -# Source other necessary files. -# -#----------------------------------------------------------------------- -# -. $USHDIR/link_fix.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -581,9 +573,9 @@ cd_vrfy - # #----------------------------------------------------------------------- # -link_fix \ - verbose="$VERBOSE" \ - file_group="orog" || \ +python3 $USHDIR/link_fix.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --file-group "orog" || \ print_err_msg_exit "\ Call to function to create links to orography files failed." # diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 129cfcb75b..995b17f305 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -12,14 +12,6 @@ # #----------------------------------------------------------------------- # -# Source other necessary files. -# -#----------------------------------------------------------------------- -# -. $USHDIR/link_fix.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -241,9 +233,9 @@ esac # #----------------------------------------------------------------------- # -link_fix \ - verbose="$VERBOSE" \ - file_group="sfc_climo" || \ +python3 $USHDIR/link_fix.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --file-group "sfc_climo" || \ print_err_msg_exit "\ Call to function to create links to surface climatology files failed." # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 8593f9c536..18b05d5c3c 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -9,16 +9,6 @@ # . ${GLOBAL_VAR_DEFNS_FP} . $USHDIR/source_util_funcs.sh -. $USHDIR/set_FV3nml_ens_stoch_seeds.sh -# -#----------------------------------------------------------------------- -# -# Source other necessary files. -# -#----------------------------------------------------------------------- -# -. $USHDIR/create_model_configure_file.sh -. $USHDIR/create_diag_table_file.sh # #----------------------------------------------------------------------- # @@ -446,7 +436,9 @@ fi if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] \ [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then - set_FV3nml_ens_stoch_seeds cdate="$cdate" || print_err_msg_exit "\ + python3 $USHDIR/set_FV3nml_ens_stoch_seeds.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --cdate "$cdate" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" @@ -464,12 +456,13 @@ fi # #----------------------------------------------------------------------- # -create_model_configure_file \ - cdate="$cdate" \ - run_dir="${run_dir}" \ - sub_hourly_post="${SUB_HOURLY_POST}" \ - dt_subhourly_post_mnts="${DT_SUBHOURLY_POST_MNTS}" \ - dt_atmos="${DT_ATMOS}" || print_err_msg_exit "\ +python3 $USHDIR/create_model_configure_file.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --cdate "$cdate" \ + --run-dir "${run_dir}" \ + --sub-hourly-post "${SUB_HOURLY_POST}" \ + --dt-subhourly-post-mnts "${DT_SUBHOURLY_POST_MNTS}" \ + --dt-atmos "${DT_ATMOS}" || print_err_msg_exit "\ Call to function to create a model configuration file for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" @@ -482,8 +475,9 @@ cycle's (cdate) run directory (run_dir) failed: # #----------------------------------------------------------------------- # -create_diag_table_file \ - run_dir="${run_dir}" || print_err_msg_exit "\ +python3 $USHDIR/create_diag_table_file.py \ + --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ + --run-dir "${run_dir}" || print_err_msg_exit "\ Call to function to create a diag table file for the current cycle's (cdate) run directory (run_dir) failed: run_dir = \"${run_dir}\"" diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 39dc68f516..23ef0500c2 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -438,9 +438,6 @@ function get_WE2Etest_names_subdirs_descs() { # ushdir=$( readlink -f "$WE2Edir/../../ush" ) . $ushdir/constants.sh - . $ushdir/set_predef_grid_params.sh - . $ushdir/set_gridparams_GFDLgrid.sh - . $ushdir/set_gridparams_ESGgrid.sh # #----------------------------------------------------------------------- # @@ -1220,70 +1217,16 @@ ${test_desc}${stripped_line} " # parameters. The way the latter are obtained depends on whether or not # a predefined grid is being used. # -# If using a predefined grid, call the set_predef_grid_params() function -# to get the grid parameters. -# - if [ ! -z "${PREDEF_GRID_NAME}" ]; then -# -# Note: -# Can set "quilting" to "FALSE" in the following argument list because -# the write-component parameters are not needed below; only those of the -# native grid are needed. -# - set_predef_grid_params \ - predef_grid_name="${PREDEF_GRID_NAME}" \ - quilting="FALSE" \ - outvarname_grid_gen_method="grid_gen_method" \ - outvarname_esggrid_lon_ctr="esggrid_lon_ctr" \ - outvarname_esggrid_lat_ctr="esggrid_lat_ctr" \ - outvarname_esggrid_delx="esggrid_delx" \ - outvarname_esggrid_dely="esggrid_dely" \ - outvarname_esggrid_nx="esggrid_nx" \ - outvarname_esggrid_ny="esggrid_ny" \ - outvarname_esggrid_pazi="esggrid_pazi" \ - outvarname_esggrid_wide_halo_width="esggrid_wide_halo_width" \ - outvarname_gfdlgrid_lon_t6_ctr="gfdlgrid_lon_t6_ctr" \ - outvarname_gfdlgrid_lat_t6_ctr="gfdlgrid_lat_t6_ctr" \ - outvarname_gfdlgrid_stretch_fac="gfdlgrid_stretch_fac" \ - outvarname_gfdlgrid_num_cells="gfdlgrid_num_cells" \ - outvarname_gfdlgrid_refine_ratio="gfdlgrid_refine_ratio" \ - outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="gfdlgrid_istart_of_rgnl_dom_on_t6g" \ - outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="gfdlgrid_iend_of_rgnl_dom_on_t6g" \ - outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g="gfdlgrid_jstart_of_rgnl_dom_on_t6g" \ - outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g="gfdlgrid_jend_of_rgnl_dom_on_t6g" \ - outvarname_dt_atmos="dta" -# -# If using a custom grid, the test's configuration file should contain -# the grid parameters. Source this file and set the values of the grid -# parameters it contains to local variables. -# - else - - . ./${config_fn} - grid_gen_method="${GRID_GEN_METHOD}" - if [ "${grid_gen_method}" = "GFDLgrid" ]; then - gfdlgrid_lon_t6_ctr="${GFDLgrid_LON_T6_CTR}" - gfdlgrid_lat_t6_ctr="${GFDLgrid_LAT_T6_CTR}" - gfdlgrid_num_cells="${GFDLgrid_NUM_CELLS}" - gfdlgrid_stretch_fac="${GFDLgrid_STRETCH_FAC}" - gfdlgrid_refine_ratio="${GFDLgrid_REFINE_RATIO}" - gfdlgrid_istart_of_rgnl_dom_on_t6g="${GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G}" - gfdlgrid_iend_of_rgnl_dom_on_t6g="${GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G}" - gfdlgrid_jstart_of_rgnl_dom_on_t6g="${GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G}" - gfdlgrid_jend_of_rgnl_dom_on_t6g="${GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G}" - elif [ "${grid_gen_method}" = "ESGgrid" ]; then - esggrid_lon_ctr="${ESGgrid_LON_CTR}" - esggrid_lat_ctr="${ESGgrid_LAT_CTR}" - esggrid_delx="${ESGgrid_DELX}" - esggrid_dely="${ESGgrid_DELY}" - esggrid_nx="${ESGgrid_NX}" - esggrid_ny="${ESGgrid_NY}" - esggrid_pazi="${ESGgrid_PAZI}" - esggrid_wide_halo_width="${ESGgrid_WIDE_HALO_WIDTH}" - fi - dta="${DT_ATMOS}" +params=$(\ + PREDEF_GRID_NAME="${PREDEF_GRID_NAME}" \ + QUILTING="FALSE" \ + RADIUS_EARTH=${RADIUS_EARTH} \ + DEGS_PER_RADIAN=${DEGS_PER_RADIAN} \ + NH4=${NH4} \ + $ushdir/calculate_cost.py -c "${test_configs_basedir}/$subdir/${config_fn}") + +read dta nxny dta_r nxny_r <<< "${params}" - fi # # Save the value of dta (which is just dt_atmos) in an array. The single # quote at the beginning forces Google Sheets to interpret this quantity @@ -1292,49 +1235,9 @@ ${test_desc}${stripped_line} " # prim_test_dt_atmos+=( "'${dta}" ) # -# The way the number of grid points in the horizontal directions (nx and -# ny) are calculated depends on the method used to generate the grid as -# well as the grid parameters for that method. -# - if [ "${grid_gen_method}" = "GFDLgrid" ]; then -# -# Note: -# The workflow generation mode (run_envir) can be set to "community" here -# since this does not affect the values of nx and ny. -# - set_gridparams_GFDLgrid \ - lon_of_t6_ctr="${gfdlgrid_lon_t6_ctr}" \ - lat_of_t6_ctr="${gfdlgrid_lat_t6_ctr}" \ - res_of_t6g="${gfdlgrid_num_cells}" \ - stretch_factor="${gfdlgrid_stretch_fac}" \ - refine_ratio_t6g_to_t7g="${gfdlgrid_refine_ratio}" \ - istart_of_t7_on_t6g="${gfdlgrid_istart_of_rgnl_dom_on_t6g}" \ - iend_of_t7_on_t6g="${gfdlgrid_iend_of_rgnl_dom_on_t6g}" \ - jstart_of_t7_on_t6g="${gfdlgrid_jstart_of_rgnl_dom_on_t6g}" \ - jend_of_t7_on_t6g="${gfdlgrid_jend_of_rgnl_dom_on_t6g}" \ - verbose="$verbose" \ - outvarname_nx_of_t7_on_t7g="nx" \ - outvarname_ny_of_t7_on_t7g="ny" - - elif [ "${grid_gen_method}" = "ESGgrid" ]; then - - set_gridparams_ESGgrid \ - lon_ctr="${esggrid_lon_ctr}" \ - lat_ctr="${esggrid_lat_ctr}" \ - nx="${esggrid_nx}" \ - ny="${esggrid_ny}" \ - pazi="${esggrid_pazi}" \ - halo_width="${esggrid_wide_halo_width}" \ - delx="${esggrid_delx}" \ - dely="${esggrid_dely}" \ - outvarname_nx="nx" \ - outvarname_ny="ny" - - fi -# # Calculate the total number of horizontal grid points. # - num_grid_pts=$(( nx*ny )) + num_grid_pts=$nxny # # Calculate the number of time steps for the test. Note that FCST_LEN_HRS # is in units of hours while dta is in units of seconds. Also, the factor @@ -1348,30 +1251,6 @@ ${test_desc}${stripped_line} " # ac=$(( num_grid_pts*num_time_steps*nf )) # -# Unset all grid paramters so that they are not accidentally reused for -# the next test. -# - unset gfdlgrid_lon_t6_ctr \ - gfdlgrid_lat_t6_ctr \ - gfdlgrid_num_cells \ - gfdlgrid_stretch_fac \ - gfdlgrid_refine_ratio \ - gfdlgrid_istart_of_rgnl_dom_on_t6g \ - gfdlgrid_iend_of_rgnl_dom_on_t6g \ - gfdlgrid_jstart_of_rgnl_dom_on_t6g \ - gfdlgrid_jend_of_rgnl_dom_on_t6g \ - esggrid_lon_ctr \ - esggrid_lat_ctr \ - esggrid_nx \ - esggrid_ny \ - esggrid_pazi \ - esggrid_wide_halo_width \ - esggrid_delx \ - esggrid_dely \ - dta \ - nx \ - ny -# # Save the absolute cost for this test in the array that will eventually # contain the relative cost. The values in this array will be divided # by abs_cost_ref later below to obtain relative costs. @@ -1397,16 +1276,9 @@ ${test_desc}${stripped_line} " # #----------------------------------------------------------------------- # - set_predef_grid_params \ - predef_grid_name="RRFS_CONUS_25km" \ - quilting="FALSE" \ - outvarname_esggrid_nx="nx" \ - outvarname_esggrid_ny="ny" \ - outvarname_dt_atmos="dta" - - num_grid_pts=$(( nx*ny )) + num_grid_pts=$nxny_r fcst_len_hrs="6" - num_time_steps=$(( (fcst_len_hrs*3600 + dta - 1)/dta )) + num_time_steps=$(( (fcst_len_hrs*3600 + dta_r - 1)/dta_r )) abs_cost_ref=$(( num_grid_pts*num_time_steps )) for (( i=0; i<=$((num_prim_tests-1)); i++ )); do diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index b2c8c537c7..605a9178a3 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -1281,7 +1281,7 @@ exist or is not a directory: # #----------------------------------------------------------------------- # - $ushdir/generate_FV3LAM_wflow.sh || \ + $ushdir/generate_FV3LAM_wflow.py || \ print_err_msg_exit "\ Could not generate an experiment for the test specified by test_name: test_name = \"${test_name}\"" diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index de6cdf0c24..5b084c2fc6 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -79,6 +79,7 @@ exec_subdir='bin_intel/bin' env_path="${SRW_APP_DIR}/modulefiles" env_file="wflow_${machine}" echo "-- Load environment =>" $env_file +source ${SRW_APP_DIR}/etc/lmod-setup.sh ${machine} module use ${env_path} module load ${env_file} conda activate regional_workflow diff --git a/ush/calculate_cost.py b/ush/calculate_cost.py new file mode 100755 index 0000000000..010b892232 --- /dev/null +++ b/ush/calculate_cost.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +import os +import unittest +import argparse + +from python_utils import set_env_var, import_vars, export_vars, load_config_file, flatten_dict + +from set_predef_grid_params import set_predef_grid_params +from set_gridparams_ESGgrid import set_gridparams_ESGgrid +from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid + +def calculate_cost(config_fn): + global PREDEF_GRID_NAME, QUILTING, GRID_GEN_METHOD + + #import all environment variables + import_vars() + + #get grid config parameters (predefined or custom) + if PREDEF_GRID_NAME: + set_env_var('QUILTING',False) + set_predef_grid_params() + import_vars() + else: + cfg_u = load_config_file(config_fn) + cfg_u = flatten_dict(cfg_u) + import_vars(dictionary=cfg_u) + + #number of gridpoints (nx*ny) depends on grid generation method + if GRID_GEN_METHOD == "GFDLgrid": + (\ + LON_CTR,LAT_CTR,NX,NY,NHW,STRETCH_FAC, + ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG \ + ) = \ + set_gridparams_GFDLgrid( \ + lon_of_t6_ctr=GFDLgrid_LON_T6_CTR, \ + lat_of_t6_ctr=GFDLgrid_LAT_T6_CTR, \ + res_of_t6g=GFDLgrid_NUM_CELLS, \ + stretch_factor=GFDLgrid_STRETCH_FAC, \ + refine_ratio_t6g_to_t7g=GFDLgrid_REFINE_RATIO, \ + istart_of_t7_on_t6g=GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G, \ + iend_of_t7_on_t6g=GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G, \ + jstart_of_t7_on_t6g=GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G, \ + jend_of_t7_on_t6g=GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G) + + elif GRID_GEN_METHOD == "ESGgrid": + (\ + LON_CTR,LAT_CTR,NX,NY,PAZI, + NHW,STRETCH_FAC,DEL_ANGLE_X_SG,DEL_ANGLE_Y_SG, + NEG_NX_OF_DOM_WITH_WIDE_HALO, + NEG_NY_OF_DOM_WITH_WIDE_HALO \ + ) = \ + set_gridparams_ESGgrid( \ + lon_ctr=ESGgrid_LON_CTR, \ + lat_ctr=ESGgrid_LAT_CTR, \ + nx=ESGgrid_NX, \ + ny=ESGgrid_NY, \ + pazi=ESGgrid_PAZI, \ + halo_width=ESGgrid_WIDE_HALO_WIDTH, \ + delx=ESGgrid_DELX, \ + dely=ESGgrid_DELY) + + cost = [DT_ATMOS, NX*NY] + + #reference grid (6-hour forecast on RRFS_CONUS_25km) + PREDEF_GRID_NAME="RRFS_CONUS_25km" + + export_vars() + set_predef_grid_params() + import_vars() + cost.extend([DT_ATMOS, ESGgrid_NX*ESGgrid_NY]) + + return cost + +#interface +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=\ + 'Calculates parameters needed for calculating cost.') + parser.add_argument('--cfg','-c',dest='cfg',required=True, + help='config file containing grip params') + args = parser.parse_args() + + params = calculate_cost(args.cfg) + print(' '.join(map(str,params))) + +class Testing(unittest.TestCase): + def test_calculate_cost(self): + USHDIR = os.path.dirname(os.path.abspath(__file__)) + config_fn = os.path.join(USHDIR, "config.community.sh") + params = calculate_cost(config_fn) + self.assertCountEqual(params, [36, 1987440, 36, 28689]) + + def setUp(self): + set_env_var('DEBUG',False) + set_env_var('PREDEF_GRID_NAME',"RRFS_CONUS_3km") + set_env_var('DT_ATMOS',36) + set_env_var('LAYOUT_X',18) + set_env_var('LAYOUT_Y',36) + set_env_var('BLOCKSIZE',28) + set_env_var('QUILTING',False) + set_env_var('RADIUS_EARTH',6371200.0) + set_env_var('DEGS_PER_RADIAN',57.2957795131) + diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh deleted file mode 100644 index 35e4db195f..0000000000 --- a/ush/check_ruc_lsm.sh +++ /dev/null @@ -1,120 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that checks whether the RUC land surface -# model (LSM) parameterization is being called by the selected physics -# suite. If so, it sets the variable ruc_lsm used to "TRUE". If not, -# it sets this variable to "FALSE". It then "returns" this variable, -# i.e. it sets the environment variable whose name is specified by the -# input argument output_varname_sdf_uses_ruc_lsm to whatever sdf_uses_ruc_lsm -# is set to. -# -#----------------------------------------------------------------------- -# -function check_ruc_lsm() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "ccpp_phys_suite_fp" \ - "output_varname_sdf_uses_ruc_lsm" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local ruc_lsm_name \ - regex_search \ - ruc_lsm_name_or_null \ - sdf_uses_ruc_lsm -# -#----------------------------------------------------------------------- -# -# Check the suite definition file to see whether the Thompson microphysics -# parameterization is being used. -# -#----------------------------------------------------------------------- -# - ruc_lsm_name="lsm_ruc" - regex_search="^[ ]*(${ruc_lsm_name})<\/scheme>[ ]*$" - ruc_lsm_name_or_null=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) - - if [ "${ruc_lsm_name_or_null}" = "${ruc_lsm_name}" ]; then - sdf_uses_ruc_lsm="TRUE" - elif [ -z "${ruc_lsm_name_or_null}" ]; then - sdf_uses_ruc_lsm="FALSE" - else - print_err_msg_exit "\ -Unexpected value returned for ruc_lsm_name_or_null: - ruc_lsm_name_or_null = \"${ruc_lsm_name_or_null}\" -This variable should be set to either \"${ruc_lsm_name}\" or an empty -string." - fi -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - eval ${output_varname_sdf_uses_ruc_lsm}="${sdf_uses_ruc_lsm}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 6790d66861..6c9dc7b032 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -4,7 +4,7 @@ # This file sets the experiment's configuration variables (which are # global shell variables) to their default values. For many of these # variables, the valid values that they may take on are defined in the -# file $USHDIR/valid_param_vals.sh. +# file $USHDIR/valid_param_vals.py. # #----------------------------------------------------------------------- # @@ -45,7 +45,7 @@ RUN_ENVIR="nco" # supported platform, and you want to use the Rocoto workflow manager, # you will need set MACHINE="linux" and WORKFLOW_MANAGER="rocoto". This # combination will assume a Slurm batch manager when generating the XML. -# Please see ush/valid_param_vals.sh for a full list of supported +# Please see ush/valid_param_vals.py for a full list of supported # platforms. # # MACHINE_FILE: @@ -623,7 +623,7 @@ NDAS_OBS_DIR="" # conditions from the GFS started 6 hours earlier, then # EXTRN_MDL_LBCS_OFFSET_HRS=6. # Note: the default value is model-dependent and set in -# set_extrn_mdl_params.sh +# set_extrn_mdl_params.py # # FV3GFS_FILE_FMT_ICS: # If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS @@ -1016,7 +1016,7 @@ GFDLgrid_USE_NUM_CELLS_IN_FILENAMES="" # in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; # they are not needed by the forecast model. # NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified -# variable. Just set it in the function set_gridparams_ESGgrid.sh. +# variable. Just set it in the function set_gridparams_ESGgrid.py. # # Note that: # @@ -1176,7 +1176,7 @@ WRTCMP_dy="" # commonly used set of grid-dependent parameters. The predefined grid # parameters are specified in the script # -# $HOMErrfs/ush/set_predef_grid_params.sh +# $HOMErrfs/ush/set_predef_grid_params.py # #----------------------------------------------------------------------- # @@ -1914,6 +1914,7 @@ LSM_SPP_LSCALE=( "150000" "150000" "150000" "150000" "150000" "150000" "150000" ISEED_LSM_SPP=( "9" ) LSM_SPP_VAR_LIST=( "smc" "vgf" "alb" "sal" "emi" "zol" "stc" ) LSM_SPP_MAG_LIST=( "0.017" "0.001" "0.001" "0.001" "0.001" "0.001" "0.2" ) +LSM_SPP_EACH_STEP="true" #Sets lndp_each_step=.true. # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml deleted file mode 100644 index 6498ecf013..0000000000 --- a/ush/config_defaults.yaml +++ /dev/null @@ -1,2017 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file sets the experiment's configuration variables (which are -# global shell variables) to their default values. For many of these -# variables, the valid values that they may take on are defined in the -# file $USHDIR/valid_param_vals.sh. -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Set the RUN_ENVIR variable that is listed and described in the WCOSS -# Implementation Standards document: -# -# NCEP Central Operations -# WCOSS Implementation Standards -# April 19, 2022 -# Version 11.0.0 -# -# RUN_ENVIR is described in this document as follows: -# -# Set to "nco" if running in NCO's production environment. Used to -# distinguish between organizations. -# -# Valid values are "nco" and "community". Here, we use it to generate -# and run the experiment either in NCO mode (if RUN_ENVIR is set to "nco") -# or in community mode (if RUN_ENVIR is set to "community"). This has -# implications on the experiment variables that need to be set and the -# the directory structure used. -# -#----------------------------------------------------------------------- -# -RUN_ENVIR: "nco" -# -#----------------------------------------------------------------------- -# -# mach_doc_start -# Set machine and queue parameters. Definitions: -# -# MACHINE: -# Machine on which the workflow will run. If you are NOT on a named, -# supported platform, and you want to use the Rocoto workflow manager, -# you will need set MACHINE: "linux" and WORKFLOW_MANAGER: "rocoto". This -# combination will assume a Slurm batch manager when generating the XML. -# Please see ush/valid_param_vals.sh for a full list of supported -# platforms. -# -# MACHINE_FILE: -# Path to a configuration file with machine-specific settings. If none -# is provided, setup.sh will attempt to set the path to for a supported -# platform. -# -# ACCOUNT: -# The account under which to submit jobs to the queue. -# -# WORKFLOW_MANAGER: -# The workflow manager to use (e.g. rocoto). This is set to "none" by -# default, but if the machine name is set to a platform that supports -# rocoto, this will be overwritten and set to "rocoto". If set -# explicitly to rocoto along with the use of the MACHINE=linux target, -# the configuration layer assumes a Slurm batch manager when generating -# the XML. Valid options: "rocoto" or "none" -# -# NCORES_PER_NODE: -# The number of cores available per node on the compute platform. Set -# for supported platforms in setup.sh, but is now also configurable for -# all platforms. -# -# LMOD_PATH: -# Path to the LMOD sh file on your Linux system. Is set automatically -# for supported machines. -# -# BUILD_MOD_FN: -# Name of alternative build module file to use if using an -# unsupported platform. Is set automatically for supported machines. -# -# WFLOW_MOD_FN: -# Name of alternative workflow module file to use if using an -# unsupported platform. Is set automatically for supported machines. -# -# SCHED: -# The job scheduler to use (e.g. slurm). Set this to an empty string in -# order for the experiment generation script to set it depending on the -# machine. -# -# PARTITION_DEFAULT: -# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), -# the default partition to which to submit workflow tasks. If a task -# does not have a specific variable that specifies the partition to which -# it will be submitted (e.g. PARTITION_HPSS, PARTITION_FCST; see below), -# it will be submitted to the partition specified by this variable. If -# this is not set or is set to an empty string, it will be (re)set to a -# machine-dependent value. This is not used if SCHED is not set to -# "slurm". -# -# QUEUE_DEFAULT: -# The default queue or QOS (if using the slurm job scheduler, where QOS -# is Quality of Service) to which workflow tasks are submitted. If a -# task does not have a specific variable that specifies the queue to which -# it will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will -# be submitted to the queue specified by this variable. If this is not -# set or is set to an empty string, it will be (re)set to a machine- -# dependent value. -# -# PARTITION_HPSS: -# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), -# the partition to which the tasks that get or create links to external -# model files [which are needed to generate initial conditions (ICs) and -# lateral boundary conditions (LBCs)] are submitted. If this is not set -# or is set to an empty string, it will be (re)set to a machine-dependent -# value. This is not used if SCHED is not set to "slurm". -# -# QUEUE_HPSS: -# The queue or QOS to which the tasks that get or create links to external -# model files [which are needed to generate initial conditions (ICs) and -# lateral boundary conditions (LBCs)] are submitted. If this is not set -# or is set to an empty string, it will be (re)set to a machine-dependent -# value. -# -# PARTITION_FCST: -# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), -# the partition to which the task that runs forecasts is submitted. If -# this is not set or set to an empty string, it will be (re)set to a -# machine-dependent value. This is not used if SCHED is not set to -# "slurm". -# -# QUEUE_FCST: -# The queue or QOS to which the task that runs a forecast is submitted. -# If this is not set or set to an empty string, it will be (re)set to a -# machine-dependent value. -# -# mach_doc_end -# -#----------------------------------------------------------------------- -# -MACHINE: "BIG_COMPUTER" -MACHINE_FILE: "" -ACCOUNT: "project_name" -WORKFLOW_MANAGER: "none" -NCORES_PER_NODE: "" -LMOD_PATH: "" -BUILD_MOD_FN: "" -WFLOW_MOD_FN: "" -SCHED: "" -PARTITION_DEFAULT: "" -QUEUE_DEFAULT: "" -PARTITION_HPSS: "" -QUEUE_HPSS: "" -PARTITION_FCST: "" -QUEUE_FCST: "" -# -#----------------------------------------------------------------------- -# -# Set run commands for platforms without a workflow manager. These values -# will be ignored unless WORKFLOW_MANAGER: "none". Definitions: -# -# RUN_CMD_UTILS: -# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, -# etc.) Can be left blank for smaller domains, in which case the executables -# will run without MPI. -# -# RUN_CMD_FCST: -# The run command for the model forecast step. This will be appended to -# the end of the variable definitions file, so it can reference other -# variables. -# -# RUN_CMD_POST: -# The run command for post-processing (UPP). Can be left blank for smaller -# domains, in which case UPP will run without MPI. -# -#----------------------------------------------------------------------- -# -RUN_CMD_UTILS: "mpirun -np 1" -RUN_CMD_FCST: "mpirun -np ${PE_MEMBER01}" -RUN_CMD_POST: "mpirun -np 1" -# -#----------------------------------------------------------------------- -# -# Set cron-associated parameters. Definitions: -# -# USE_CRON_TO_RELAUNCH: -# Flag that determines whether or not to add a line to the user's cron -# table to call the experiment launch script every CRON_RELAUNCH_INTVL_MNTS -# minutes. -# -# CRON_RELAUNCH_INTVL_MNTS: -# The interval (in minutes) between successive calls of the experiment -# launch script by a cron job to (re)launch the experiment (so that the -# workflow for the experiment kicks off where it left off). -# -#----------------------------------------------------------------------- -# -USE_CRON_TO_RELAUNCH: "FALSE" -CRON_RELAUNCH_INTVL_MNTS: "03" -# -#----------------------------------------------------------------------- -# -# dir_doc_start -# Set directories. Definitions: -# -# EXPT_BASEDIR: -# The base directory in which the experiment directory will be created. -# If this is not specified or if it is set to an empty string, it will -# default to ${HOMErrfs}/../expt_dirs. -# -# EXPT_SUBDIR: -# The name that the experiment directory (without the full path) will -# have. The full path to the experiment directory, which will be contained -# in the variable EXPTDIR, will be: -# -# EXPTDIR: "${EXPT_BASEDIR}/${EXPT_SUBDIR}" -# -# This cannot be empty. If set to a null string here, it must be set to -# a (non-empty) value in the user-defined experiment configuration file. -# -# dir_doc_end -# -# EXEC_SUBDIR: -# The name of the subdirectory of ufs-srweather-app where executables are -# installed. -#----------------------------------------------------------------------- -# -EXPT_BASEDIR: "" -EXPT_SUBDIR: "" -EXEC_SUBDIR: "bin" -# -#----------------------------------------------------------------------- -# -# Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is -# set to "nco"). Definitions: -# -# COMIN: -# Directory containing files generated by the external model (FV3GFS, NAM, -# HRRR, etc) that the initial and lateral boundary condition generation tasks -# need in order to create initial and boundary condition files for a given -# cycle on the native FV3-LAM grid. -# -# envir, NET, model_ver, RUN: -# Standard environment variables defined in the NCEP Central Operations WCOSS -# Implementation Standards document as follows: -# -# envir: -# Set to "test" during the initial testing phase, "para" when running -# in parallel (on a schedule), and "prod" in production. -# -# NET: -# Model name (first level of com directory structure) -# -# model_ver: -# Version number of package in three digits (second level of com directory) -# -# RUN: -# Name of model run (third level of com directory structure). -# In general, same as $NET -# -# STMP: -# The beginning portion of the directory that will contain cycle-dependent -# model input files, symlinks to cycle-independent input files, and raw -# (i.e. before post-processing) forecast output files for a given cycle. -# For a cycle that starts on the date specified by yyyymmdd and hour -# specified by hh (where yyyymmdd and hh are as described above) [so that -# the cycle date (cdate) is given by cdate: "${yyyymmdd}${hh}"], the -# directory in which the aforementioned files will be located is: -# -# $STMP/tmpnwprd/$RUN/$cdate -# -# PTMP: -# The beginning portion of the directory that will contain the output -# files from the post-processor (UPP) for a given cycle. For a cycle -# that starts on the date specified by yyyymmdd and hour specified by hh -# (where yyyymmdd and hh are as described above), the directory in which -# the UPP output files will be placed will be: -# -# $PTMP/com/$NET/$model_ver/$RUN.$yyyymmdd/$hh -# -#----------------------------------------------------------------------- -# -COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS" -envir: "para" -NET: "rrfs" -model_ver: "v1.0.0" -RUN: "rrfs" -STMP: "/base/path/of/directory/containing/model/input/and/raw/output/files" -PTMP: "/base/path/of/directory/containing/postprocessed/output/files" -# -#----------------------------------------------------------------------- -# -# Set the separator character(s) to use in the names of the grid, mosaic, -# and orography fixed files. -# -# Ideally, the same separator should be used in the names of these fixed -# files as the surface climatology fixed files (which always use a "." -# as the separator), i.e. ideally, DOT_OR_USCORE should be set to "." -# -#----------------------------------------------------------------------- -# -DOT_OR_USCORE: "_" -# -#----------------------------------------------------------------------- -# -# Set file names. Definitions: -# -# EXPT_CONFIG_FN: -# Name of the user-specified configuration file for the forecast experiment. -# -# RGNL_GRID_NML_FN: -# Name of file containing the namelist settings for the code that generates -# a "ESGgrid" type of regional grid. -# -# FV3_NML_BASE_SUITE_FN: -# Name of Fortran namelist file containing the forecast model's base suite -# namelist, i.e. the portion of the namelist that is common to all physics -# suites. -# -# FV3_NML_YAML_CONFIG_FN: -# Name of YAML configuration file containing the forecast model's namelist -# settings for various physics suites. -# -# FV3_NML_BASE_ENS_FN: -# Name of Fortran namelist file containing the forecast model's base -# ensemble namelist, i.e. the the namelist file that is the starting point -# from which the namelist files for each of the enesemble members are -# generated. -# -# FV3_EXEC_FN: -# Name to use for the forecast model executable when it is copied from -# the directory in which it is created in the build step to the executables -# directory (EXECDIR; this is set during experiment generation). -# -# DIAG_TABLE_TMPL_FN: -# Name of a template file that specifies the output fields of the forecast -# model (ufs-weather-model: diag_table) followed by [dot_ccpp_phys_suite]. -# Its default value is the name of the file that the ufs weather model -# expects to read in. -# -# FIELD_TABLE_TMPL_FN: -# Name of a template file that specifies the tracers in IC/LBC files of the -# forecast model (ufs-weather-mode: field_table) followed by [dot_ccpp_phys_suite]. -# Its default value is the name of the file that the ufs weather model expects -# to read in. -# -# MODEL_CONFIG_TMPL_FN: -# Name of a template file that contains settings and configurations for the -# NUOPC/ESMF main component (ufs-weather-model: model_config). Its default -# value is the name of the file that the ufs weather model expects to read in. -# -# NEMS_CONFIG_TMPL_FN: -# Name of a template file that contains information about the various NEMS -# components and their run sequence (ufs-weather-model: nems.configure). -# Its default value is the name of the file that the ufs weather model expects -# to read in. -# -# FCST_MODEL: -# Name of forecast model (default=ufs-weather-model) -# -# WFLOW_XML_FN: -# Name of the rocoto workflow XML file that the experiment generation -# script creates and that defines the workflow for the experiment. -# -# GLOBAL_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of the primary -# experiment variables (parameters) defined in this default configuration -# script and in the user-specified configuration as well as secondary -# experiment variables generated by the experiment generation script. -# This file is sourced by many scripts (e.g. the J-job scripts corresponding -# to each workflow task) in order to make all the experiment variables -# available in those scripts. -# -# EXTRN_MDL_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which ICs or LBCs are generated. This -# file is created by the GET_EXTRN_*_TN task because the values of the variables -# it contains are not known before this task runs. The file is then sourced by -# the MAKE_ICS_TN and MAKE_LBCS_TN tasks. -# -# WFLOW_LAUNCH_SCRIPT_FN: -# Name of the script that can be used to (re)launch the experiment's rocoto -# workflow. -# -# WFLOW_LAUNCH_LOG_FN: -# Name of the log file that contains the output from successive calls to -# the workflow launch script (WFLOW_LAUNCH_SCRIPT_FN). -# -#----------------------------------------------------------------------- -# -EXPT_CONFIG_FN: "config.sh" - -RGNL_GRID_NML_FN: "regional_grid.nml" - -FV3_NML_BASE_SUITE_FN: "input.nml.FV3" -FV3_NML_YAML_CONFIG_FN: "FV3.input.yml" -FV3_NML_BASE_ENS_FN: "input.nml.base_ens" -FV3_EXEC_FN: "ufs_model" - -DATA_TABLE_TMPL_FN: "" -DIAG_TABLE_TMPL_FN: "" -FIELD_TABLE_TMPL_FN: "" -MODEL_CONFIG_TMPL_FN: "" -NEMS_CONFIG_TMPL_FN: "" - -FCST_MODEL: "ufs-weather-model" -WFLOW_XML_FN: "FV3LAM_wflow.xml" -GLOBAL_VAR_DEFNS_FN: "var_defns.sh" -EXTRN_MDL_VAR_DEFNS_FN: "extrn_mdl_var_defns.sh" -WFLOW_LAUNCH_SCRIPT_FN: "launch_FV3LAM_wflow.sh" -WFLOW_LAUNCH_LOG_FN: "log.launch_FV3LAM_wflow" -# -#----------------------------------------------------------------------- -# -# Set output file name. Definitions: -# -# POST_OUTPUT_DOMAIN_NAME: -# Domain name used in naming the output files of run_post by UPP or inline post. -# Output file name: $NET.tHHz.[var_name].f###.$POST_OUTPUT_DOMAIN_NAME.grib2 -# -#----------------------------------------------------------------------- -# -POST_OUTPUT_DOMAIN_NAME: "" -# -#----------------------------------------------------------------------- -# -# Set forecast parameters. Definitions: -# -# DATE_FIRST_CYCL: -# Starting date of the first forecast in the set of forecasts to run. -# Format is "YYYYMMDD". Note that this does not include the hour-of-day. -# -# DATE_LAST_CYCL: -# Starting date of the last forecast in the set of forecasts to run. -# Format is "YYYYMMDD". Note that this does not include the hour-of-day. -# -# CYCL_HRS: -# An array containing the hours of the day at which to launch forecasts. -# Forecasts are launched at these hours on each day from DATE_FIRST_CYCL -# to DATE_LAST_CYCL, inclusive. Each element of this array must be a -# two-digit string representing an integer that is less than or equal to -# 23, e.g. "00", "03", "12", "23". -# -# INCR_CYCL_FREQ: -# Increment in hours for Cycle Frequency (cycl_freq). -# Default is 24, which means cycle_freq=24:00:00 -# -# FCST_LEN_HRS: -# The length of each forecast, in integer hours. -# -#----------------------------------------------------------------------- -# -DATE_FIRST_CYCL: "YYYYMMDD" -DATE_LAST_CYCL: "YYYYMMDD" -CYCL_HRS: [ "HH1", "HH2" ] -INCR_CYCL_FREQ: "24" -FCST_LEN_HRS: "24" -# -#----------------------------------------------------------------------- -# -# Set model_configure parameters. Definitions: -# -# DT_ATMOS: -# The main forecast model integraton time step. As described in the -# forecast model documentation, "It corresponds to the frequency with -# which the top level routine in the dynamics is called as well as the -# frequency with which the physics is called." -# -# CPL: parameter for coupling -# (set automatically based on FCST_MODEL in ush/setup.sh) -# (ufs-weather-model:FALSE, fv3gfs_aqm:TRUE) -# -# RESTART_INTERVAL: -# frequency of the output restart files (unit:hour). -# Default=0: restart files are produced at the end of a forecast run -# For example, RESTART_INTERVAL: "1": restart files are produced every hour -# with the prefix "YYYYMMDD.HHmmSS." in the RESTART directory -# -# WRITE_DOPOST: -# Flag that determines whether or not to use the INLINE POST option -# When TRUE, force to turn off run_post (RUN_TASK_RUN_POST=FALSE) in setup.sh -# -#----------------------------------------------------------------------- -# -DT_ATMOS: "" -RESTART_INTERVAL: "0" -WRITE_DOPOST: "FALSE" -# -#----------------------------------------------------------------------- -# -# Set METplus parameters. Definitions: -# -# MODEL: -# String that specifies a descriptive name for the model being verified. -# -# MET_INSTALL_DIR: -# Location to top-level directory of MET installation. -# -# METPLUS_PATH: -# Location to top-level directory of METplus installation. -# -# CCPA_OBS_DIR: -# User-specified location of top-level directory where CCPA hourly -# precipitation files used by METplus are located. This parameter needs -# to be set for both user-provided observations and for observations -# that are retrieved from the NOAA HPSS (if the user has access) via -# the get_obs_ccpa_tn task (activated in workflow by setting -# RUN_TASK_GET_OBS_CCPA="TRUE"). In the case of pulling observations -# directly from NOAA HPSS, the data retrieved will be placed in this -# directory. Please note, this path must be defind as -# /full-path-to-obs/ccpa/proc. METplus is configured to verify 01-, -# 03-, 06-, and 24-h accumulated precipitation using hourly CCPA files. -# METplus configuration files require the use of predetermined directory -# structure and file names. Therefore, if the CCPA files are user -# provided, they need to follow the anticipated naming structure: -# {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, where YYYY is the 4-digit -# valid year, MM the 2-digit valid month, DD the 2-digit valid day of -# the month, and HH the 2-digit valid hour of the day. In addition, a -# caveat is noted for using hourly CCPA data. There is a problem with -# the valid time in the metadata for files valid from 19 - 00 UTC (or -# files under the '00' directory). The script to pull the CCPA data -# from the NOAA HPSS has an example of how to account for this as well -# as organizing the data into a more intuitive format: -# regional_workflow/scripts/exregional_get_ccpa_files.sh. When a fix -# is provided, it will be accounted for in the -# exregional_get_ccpa_files.sh script. -# -# MRMS_OBS_DIR: -# User-specified location of top-level directory where MRMS composite -# reflectivity files used by METplus are located. This parameter needs -# to be set for both user-provided observations and for observations -# that are retrieved from the NOAA HPSS (if the user has access) via the -# get_obs_mrms_tn task (activated in workflow by setting -# RUN_TASK_GET_OBS_MRMS="TRUE"). In the case of pulling observations -# directly from NOAA HPSS, the data retrieved will be placed in this -# directory. Please note, this path must be defind as -# /full-path-to-obs/mrms/proc. METplus configuration files require the -# use of predetermined directory structure and file names. Therefore, if -# the MRMS files are user provided, they need to follow the anticipated -# naming structure: -# {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, -# where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD -# the 2-digit valid day of the month, HH the 2-digit valid hour of the -# day, mm the 2-digit valid minutes of the hour, and SS is the two-digit -# valid seconds of the hour. In addition, METplus is configured to look -# for a MRMS composite reflectivity file for the valid time of the -# forecast being verified; since MRMS composite reflectivity files do -# not always exactly match the valid time, a script, within the main -# script to retrieve MRMS data from the NOAA HPSS, is used to identify -# and rename the MRMS composite reflectivity file to match the valid -# time of the forecast. The script to pull the MRMS data from the NOAA -# HPSS has an example of the expected file naming structure: -# regional_workflow/scripts/exregional_get_mrms_files.sh. This script -# calls the script used to identify the MRMS file closest to the valid -# time: regional_workflow/ush/mrms_pull_topofhour.py. -# -# NDAS_OBS_DIR: -# User-specified location of top-level directory where NDAS prepbufr -# files used by METplus are located. This parameter needs to be set for -# both user-provided observations and for observations that are -# retrieved from the NOAA HPSS (if the user has access) via the -# get_obs_ndas_tn task (activated in workflow by setting  -# RUN_TASK_GET_OBS_NDAS="TRUE"). In the case of pulling observations -# directly from NOAA HPSS, the data retrieved will be placed in this -# directory. Please note, this path must be defind as -# /full-path-to-obs/ndas/proc. METplus is configured to verify -# near-surface variables hourly and upper-air variables at times valid -# at 00 and 12 UTC with NDAS prepbufr files. METplus configuration files -# require the use of predetermined file names. Therefore, if the NDAS -# files are user provided, they need to follow the anticipated naming -# structure: prepbufr.ndas.{YYYYMMDDHH}, where YYYY is the 4-digit valid -# year, MM the 2-digit valid month, DD the 2-digit valid day of the -# month, and HH the 2-digit valid hour of the day. The script to pull -# the NDAS data from the NOAA HPSS has an example of how to rename the -# NDAS data into a more intuitive format with the valid time listed in -# the file name: regional_workflow/scripts/exregional_get_ndas_files.sh -# -#----------------------------------------------------------------------- -# -MODEL: "" -MET_INSTALL_DIR: "" -MET_BIN_EXEC: "bin" -METPLUS_PATH: "" -CCPA_OBS_DIR: "" -MRMS_OBS_DIR: "" -NDAS_OBS_DIR: "" -# -#----------------------------------------------------------------------- -# -# Set initial and lateral boundary condition generation parameters. -# Definitions: -# -# EXTRN_MDL_NAME_ICS: -#`The name of the external model that will provide fields from which -# initial condition (including and surface) files will be generated for -# input into the forecast model. -# -# EXTRN_MDL_NAME_LBCS: -#`The name of the external model that will provide fields from which -# lateral boundary condition (LBC) files will be generated for input into -# the forecast model. -# -# LBC_SPEC_INTVL_HRS: -# The interval (in integer hours) with which LBC files will be generated. -# We will refer to this as the boundary update interval. Note that the -# model specified in EXTRN_MDL_NAME_LBCS must have data available at a -# frequency greater than or equal to that implied by LBC_SPEC_INTVL_HRS. -# For example, if LBC_SPEC_INTVL_HRS is set to 6, then the model must have -# data availble at least every 6 hours. It is up to the user to ensure -# that this is the case. -# -# EXTRN_MDL_ICS_OFFSET_HRS: -# Users may wish to start a forecast from a forecast of a previous cycle -# of an external model. This variable sets the number of hours earlier -# the external model started than when the FV3 forecast configured here -# should start. For example, the forecast should start from a 6 hour -# forecast of the GFS, then EXTRN_MDL_ICS_OFFSET_HRS=6. - -# EXTRN_MDL_LBCS_OFFSET_HRS: -# Users may wish to use lateral boundary conditions from a forecast that -# was started earlier than the initial time for the FV3 forecast -# configured here. This variable sets the number of hours earlier -# the external model started than when the FV3 forecast configured here -# should start. For example, the forecast should use lateral boundary -# conditions from the GFS started 6 hours earlier, then -# EXTRN_MDL_LBCS_OFFSET_HRS=6. -# Note: the default value is model-dependent and set in -# set_extrn_mdl_params.sh -# -# FV3GFS_FILE_FMT_ICS: -# If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS -# is set to "FV3GFS"), this variable specifies the format of the model -# files to use when generating the ICs. -# -# FV3GFS_FILE_FMT_LBCS: -# If using the FV3GFS model as the source of the LBCs (i.e. if -# EXTRN_MDL_NAME_LBCS is set to "FV3GFS"), this variable specifies the -# format of the model files to use when generating the LBCs. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_NAME_ICS: "FV3GFS" -EXTRN_MDL_NAME_LBCS: "FV3GFS" -LBC_SPEC_INTVL_HRS: "6" -EXTRN_MDL_ICS_OFFSET_HRS: "0" -EXTRN_MDL_LBCS_OFFSET_HRS: "" -FV3GFS_FILE_FMT_ICS: "nemsio" -FV3GFS_FILE_FMT_LBCS: "nemsio" -# -#----------------------------------------------------------------------- -# -# Base directories in which to search for external model files. -# -# EXTRN_MDL_SYSBASEDIR_ICS: -# Base directory on the local machine containing external model files for -# generating ICs on the native grid. The way the full path containing -# these files is constructed depends on the user-specified external model -# for ICs, i.e. EXTRN_MDL_NAME_ICS. -# -# EXTRN_MDL_SYSBASEDIR_LBCS: -# Same as EXTRN_MDL_SYSBASEDIR_ICS but for LBCs. -# -# Note that these must be defined as null strings here so that if they -# are specified by the user in the experiment configuration file, they -# remain set to those values, and if not, they get set to machine-dependent -# values. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_ICS: '' -EXTRN_MDL_SYSBASEDIR_LBCS: '' -# -#----------------------------------------------------------------------- -# -# User-staged external model directories and files. Definitions: -# -# USE_USER_STAGED_EXTRN_FILES: -# Flag that determines whether or not the workflow will look for the -# external model files needed for generating ICs and LBCs in user-specified -# directories. -# -# EXTRN_MDL_SOURCE_BASEDIR_ICS: -# Directory in which to look for external model files for generating ICs. -# If USE_USER_STAGED_EXTRN_FILES is set to "TRUE", the workflow looks in -# this directory (specifically, in a subdirectory under this directory -# named "YYYYMMDDHH" consisting of the starting date and cycle hour of -# the forecast, where YYYY is the 4-digit year, MM the 2-digit month, DD -# the 2-digit day of the month, and HH the 2-digit hour of the day) for -# the external model files specified by the array EXTRN_MDL_FILES_ICS -# (these files will be used to generate the ICs on the native FV3-LAM -# grid). This variable is not used if USE_USER_STAGED_EXTRN_FILES is -# set to "FALSE". -# -# EXTRN_MDL_FILES_ICS: -# Array containing templates of the names of the files to search for in -# the directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This -# variable is not used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". -# A single template should be used for each model file type that is -# meant to be used. You may use any of the Python-style templates -# allowed in the ush/retrieve_data.py script. To see the full list of -# supported templates, run that script with a -h option. Here is an example of -# setting FV3GFS nemsio input files: -# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio \ -# gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio ) -# Or for FV3GFS grib files: -# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} ) -# -# EXTRN_MDL_SOURCE_BASEDIR_LBCS: -# Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. -# -# EXTRN_MDL_FILES_LBCS: -# Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. -# -# EXTRN_MDL_DATA_STORES: -# A list of data stores where the scripts should look for external model -# data. The list is in priority order. If disk information is provided -# via USE_USER_STAGED_EXTRN_FILES or a known location on the platform, -# the disk location will be highest priority. Options are disk, hpss, -# aws, and nomads. -# -#----------------------------------------------------------------------- -# -USE_USER_STAGED_EXTRN_FILES: "FALSE" -EXTRN_MDL_SOURCE_BASEDIR_ICS: "" -EXTRN_MDL_FILES_ICS: "" -EXTRN_MDL_SOURCE_BASEDIR_LBCS: "" -EXTRN_MDL_FILES_LBCS: "" -EXTRN_MDL_DATA_STORES: "" -# -#----------------------------------------------------------------------- -# -# Set NOMADS online data associated parameters. Definitions: -# -# NOMADS: -# Flag controlling whether or not using NOMADS online data. -# -# NOMADS_file_type: -# Flag controlling the format of data. -# -#----------------------------------------------------------------------- -# -NOMADS: "FALSE" -NOMADS_file_type: "nemsio" -# -#----------------------------------------------------------------------- -# -# Set CCPP-associated parameters. Definitions: -# -# CCPP_PHYS_SUITE: -# The physics suite that will run using CCPP (Common Community Physics -# Package). The choice of physics suite determines the forecast model's -# namelist file, the diagnostics table file, the field table file, and -# the XML physics suite definition file that are staged in the experiment -# directory or the cycle directories under it. -# -#----------------------------------------------------------------------- -# -CCPP_PHYS_SUITE: "FV3_GFS_v16" -# -#----------------------------------------------------------------------- -# -# Set GRID_GEN_METHOD. This variable specifies the method to use to -# generate a regional grid in the horizontal. The values that it can -# take on are: -# -# * "GFDLgrid": -# This setting will generate a regional grid by first generating a -# "parent" global cubed-sphere grid and then taking a portion of tile -# 6 of that global grid -- referred to in the grid generation scripts -# as "tile 7" even though it doesn't correspond to a complete tile -- -# and using it as the regional grid. Note that the forecast is run on -# only on the regional grid (i.e. tile 7, not tiles 1 through 6). -# -# * "ESGgrid": -# This will generate a regional grid using the map projection developed -# by Jim Purser of EMC. -# -# Note that: -# -# 1) If the experiment is using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to the name of one of the valid predefined -# grids), then GRID_GEN_METHOD will be reset to the value of -# GRID_GEN_METHOD for that grid. This will happen regardless of -# whether or not GRID_GEN_METHOD is assigned a value in the user- -# specified experiment configuration file, i.e. any value it may be -# assigned in the experiment configuration file will be overwritten. -# -# 2) If the experiment is not using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to a null string), then GRID_GEN_METHOD must -# be set in the experiment configuration file. Otherwise, it will -# remain set to a null string, and the experiment generation will -# fail because the generation scripts check to ensure that it is set -# to a non-empty string before creating the experiment directory. -# -#----------------------------------------------------------------------- -# -GRID_GEN_METHOD: "" -# -#----------------------------------------------------------------------- -# -# Set parameters specific to the "GFDLgrid" method of generating a regional -# grid (i.e. for GRID_GEN_METHOD set to "GFDLgrid"). The following -# parameters will be used only if GRID_GEN_METHOD is set to "GFDLgrid". -# In this grid generation method: -# -# * The regional grid is defined with respect to a "parent" global cubed- -# sphere grid. Thus, all the parameters for a global cubed-sphere grid -# must be specified in order to define this parent global grid even -# though the model equations are not integrated on (they are integrated -# only on the regional grid). -# -# * GFDLgrid_RES is the number of grid cells in either one of the two -# horizontal directions x and y on any one of the 6 tiles of the parent -# global cubed-sphere grid. The mapping from GFDLgrid_RES to a nominal -# resolution (grid cell size) for a uniform global grid (i.e. Schmidt -# stretch factor GFDLgrid_STRETCH_FAC set to 1) for several values of -# GFDLgrid_RES is as follows: -# -# GFDLgrid_RES typical cell size -# ------------ ----------------- -# 192 50 km -# 384 25 km -# 768 13 km -# 1152 8.5 km -# 3072 3.2 km -# -# Note that these are only typical cell sizes. The actual cell size on -# the global grid tiles varies somewhat as we move across a tile. -# -# * Tile 6 has arbitrarily been chosen as the tile to use to orient the -# global parent grid on the sphere (Earth). This is done by specifying -# GFDLgrid_LON_T6_CTR and GFDLgrid_LAT_T6_CTR, which are the longitude -# and latitude (in degrees) of the center of tile 6. -# -# * Setting the Schmidt stretching factor GFDLgrid_STRETCH_FAC to a value -# greater than 1 shrinks tile 6, while setting it to a value less than -# 1 (but still greater than 0) expands it. The remaining 5 tiles change -# shape as necessary to maintain global coverage of the grid. -# -# * The cell size on a given global tile depends on both GFDLgrid_RES and -# GFDLgrid_STRETCH_FAC (since changing GFDLgrid_RES changes the number -# of cells in the tile, and changing GFDLgrid_STRETCH_FAC modifies the -# shape and size of the tile). -# -# * The regional grid is embedded within tile 6 (i.e. it doesn't extend -# beyond the boundary of tile 6). Its exact location within tile 6 is -# is determined by specifying the starting and ending i and j indices -# of the regional grid on tile 6, where i is the grid index in the x -# direction and j is the grid index in the y direction. These indices -# are stored in the variables -# -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G -# -# * In the forecast model code and in the experiment generation and workflow -# scripts, for convenience the regional grid is denoted as "tile 7" even -# though it doesn't map back to one of the 6 faces of the cube from -# which the parent global grid is generated (it maps back to only a -# subregion on face 6 since it is wholly confined within tile 6). Tile -# 6 may be referred to as the "parent" tile of the regional grid. -# -# * GFDLgrid_REFINE_RATIO is the refinement ratio of the regional grid -# (tile 7) with respect to the grid on its parent tile (tile 6), i.e. -# it is the number of grid cells along the boundary of the regional grid -# that abut one cell on tile 6. Thus, the cell size on the regional -# grid depends not only on GFDLgrid_RES and GFDLgrid_STRETCH_FAC (because -# the cell size on tile 6 depends on these two parameters) but also on -# GFDLgrid_REFINE_RATIO. Note that as on the tiles of the global grid, -# the cell size on the regional grid is not uniform but varies as we -# move across the grid. -# -# Definitions of parameters that need to be specified when GRID_GEN_METHOD -# is set to "GFDLgrid": -# -# GFDLgrid_LON_T6_CTR: -# Longitude of the center of tile 6 (in degrees). -# -# GFDLgrid_LAT_T6_CTR: -# Latitude of the center of tile 6 (in degrees). -# -# GFDLgrid_RES: -# Number of points in each of the two horizontal directions (x and y) on -# each tile of the parent global grid. Note that the name of this parameter -# is really a misnomer because although it has the stirng "RES" (for -# "resolution") in its name, it specifies number of grid cells, not grid -# size (in say meters or kilometers). However, we keep this name in order -# to remain consistent with the usage of the word "resolution" in the -# global forecast model and other auxiliary codes. -# -# GFDLgrid_STRETCH_FAC: -# Stretching factor used in the Schmidt transformation applied to the -# parent cubed-sphere grid. -# -# GFDLgrid_REFINE_RATIO: -# Cell refinement ratio for the regional grid, i.e. the number of cells -# in either the x or y direction on the regional grid (tile 7) that abut -# one cell on its parent tile (tile 6). -# -# GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: -# i-index on tile 6 at which the regional grid (tile 7) starts. -# -# GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: -# i-index on tile 6 at which the regional grid (tile 7) ends. -# -# GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: -# j-index on tile 6 at which the regional grid (tile 7) starts. -# -# GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: -# j-index on tile 6 at which the regional grid (tile 7) ends. -# -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: -# Flag that determines the file naming convention to use for grid, orography, -# and surface climatology files (or, if using pregenerated files, the -# naming convention that was used to name these files). These files -# usually start with the string "C${RES}_", where RES is an integer. -# In the global forecast model, RES is the number of points in each of -# the two horizontal directions (x and y) on each tile of the global grid -# (defined here as GFDLgrid_RES). If this flag is set to "TRUE", RES will -# be set to GFDLgrid_RES just as in the global forecast model. If it is -# set to "FALSE", we calculate (in the grid generation task) an "equivalent -# global uniform cubed-sphere resolution" -- call it RES_EQUIV -- and -# then set RES equal to it. RES_EQUIV is the number of grid points in -# each of the x and y directions on each tile that a global UNIFORM (i.e. -# stretch factor of 1) cubed-sphere grid would have to have in order to -# have the same average grid size as the regional grid. This is a more -# useful indicator of the grid size because it takes into account the -# effects of GFDLgrid_RES, GFDLgrid_STRETCH_FAC, and GFDLgrid_REFINE_RATIO -# in determining the regional grid's typical grid size, whereas simply -# setting RES to GFDLgrid_RES doesn't take into account the effects of -# GFDLgrid_STRETCH_FAC and GFDLgrid_REFINE_RATIO on the regional grid's -# resolution. Nevertheless, some users still prefer to use GFDLgrid_RES -# in the file names, so we allow for that here by setting this flag to -# "TRUE". -# -# Note that: -# -# 1) If the experiment is using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to the name of one of the valid predefined -# grids), then: -# -# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then -# these parameters will get reset to the values for that grid. -# This will happen regardless of whether or not they are assigned -# values in the user-specified experiment configuration file, i.e. -# any values they may be assigned in the experiment configuration -# file will be overwritten. -# -# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then -# these parameters will not be used and thus do not need to be reset -# to non-empty strings. -# -# 2) If the experiment is not using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to a null string), then: -# -# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified -# experiment configuration file, then these parameters must be set -# in that configuration file. -# -# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified -# experiment configuration file, then these parameters will not be -# used and thus do not need to be reset to non-empty strings. -# -#----------------------------------------------------------------------- -# -GFDLgrid_LON_T6_CTR: "" -GFDLgrid_LAT_T6_CTR: "" -GFDLgrid_RES: "" -GFDLgrid_STRETCH_FAC: "" -GFDLgrid_REFINE_RATIO: "" -GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: "" -GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G: "" -GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: "" -GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: "" -GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: "" -# -#----------------------------------------------------------------------- -# -# Set parameters specific to the "ESGgrid" method of generating a regional -# grid (i.e. for GRID_GEN_METHOD set to "ESGgrid"). Definitions: -# -# ESGgrid_LON_CTR: -# The longitude of the center of the grid (in degrees). -# -# ESGgrid_LAT_CTR: -# The latitude of the center of the grid (in degrees). -# -# ESGgrid_DELX: -# The cell size in the zonal direction of the regional grid (in meters). -# -# ESGgrid_DELY: -# The cell size in the meridional direction of the regional grid (in -# meters). -# -# ESGgrid_NX: -# The number of cells in the zonal direction on the regional grid. -# -# ESGgrid_NY: -# The number of cells in the meridional direction on the regional grid. -# -# ESGgrid_WIDE_HALO_WIDTH: -# The width (in units of number of grid cells) of the halo to add around -# the regional grid before shaving the halo down to the width(s) expected -# by the forecast model. -# -# ESGgrid_PAZI: -# The rotational parameter for the ESG grid (in degrees). -# -# In order to generate grid files containing halos that are 3-cell and -# 4-cell wide and orography files with halos that are 0-cell and 3-cell -# wide (all of which are required as inputs to the forecast model), the -# grid and orography tasks first create files with halos around the regional -# domain of width ESGgrid_WIDE_HALO_WIDTH cells. These are first stored -# in files. The files are then read in and "shaved" down to obtain grid -# files with 3-cell-wide and 4-cell-wide halos and orography files with -# 0-cell-wide (i.e. no halo) and 3-cell-wide halos. For this reason, we -# refer to the original halo that then gets shaved down as the "wide" -# halo, i.e. because it is wider than the 0-cell-wide, 3-cell-wide, and -# 4-cell-wide halos that we will eventually end up with. Note that the -# grid and orography files with the wide halo are only needed as intermediates -# in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; -# they are not needed by the forecast model. -# NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified -# variable. Just set it in the function set_gridparams_ESGgrid.sh. -# -# Note that: -# -# 1) If the experiment is using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to the name of one of the valid predefined -# grids), then: -# -# a) If the value of GRID_GEN_METHOD for that grid is "GFDLgrid", then -# these parameters will not be used and thus do not need to be reset -# to non-empty strings. -# -# b) If the value of GRID_GEN_METHOD for that grid is "ESGgrid", then -# these parameters will get reset to the values for that grid. -# This will happen regardless of whether or not they are assigned -# values in the user-specified experiment configuration file, i.e. -# any values they may be assigned in the experiment configuration -# file will be overwritten. -# -# 2) If the experiment is not using one of the predefined grids (i.e. if -# PREDEF_GRID_NAME is set to a null string), then: -# -# a) If GRID_GEN_METHOD is set to "GFDLgrid" in the user-specified -# experiment configuration file, then these parameters will not be -# used and thus do not need to be reset to non-empty strings. -# -# b) If GRID_GEN_METHOD is set to "ESGgrid" in the user-specified -# experiment configuration file, then these parameters must be set -# in that configuration file. -# -#----------------------------------------------------------------------- -# -ESGgrid_LON_CTR: "" -ESGgrid_LAT_CTR: "" -ESGgrid_DELX: "" -ESGgrid_DELY: "" -ESGgrid_NX: "" -ESGgrid_NY: "" -ESGgrid_WIDE_HALO_WIDTH: "" -ESGgrid_PAZI: "" -# -#----------------------------------------------------------------------- -# -# Set computational parameters for the forecast. Definitions: -# -# LAYOUT_X, LAYOUT_Y: -# The number of MPI tasks (processes) to use in the two horizontal -# directions (x and y) of the regional grid when running the forecast -# model. -# -# BLOCKSIZE: -# The amount of data that is passed into the cache at a time. -# -# Here, we set these parameters to null strings. This is so that, for -# any one of these parameters: -# -# 1) If the experiment is using a predefined grid, then if the user -# sets the parameter in the user-specified experiment configuration -# file (EXPT_CONFIG_FN), that value will be used in the forecast(s). -# Otherwise, the default value of the parameter for that predefined -# grid will be used. -# -# 2) If the experiment is not using a predefined grid (i.e. it is using -# a custom grid whose parameters are specified in the experiment -# configuration file), then the user must specify a value for the -# parameter in that configuration file. Otherwise, the parameter -# will remain set to a null string, and the experiment generation -# will fail because the generation scripts check to ensure that all -# the parameters defined in this section are set to non-empty strings -# before creating the experiment directory. -# -#----------------------------------------------------------------------- -# -LAYOUT_X: "" -LAYOUT_Y: "" -BLOCKSIZE: "" -# -#----------------------------------------------------------------------- -# -# Set write-component (quilting) parameters. Definitions: -# -# QUILTING: -# Flag that determines whether or not to use the write component for -# writing output files to disk. -# -# WRTCMP_write_groups: -# The number of write groups (i.e. groups of MPI tasks) to use in the -# write component. -# -# WRTCMP_write_tasks_per_group: -# The number of MPI tasks to allocate for each write group. -# -# PRINT_ESMF: -# Flag for whether or not to output extra (debugging) information from -# ESMF routines. Must be "TRUE" or "FALSE". Note that the write -# component uses ESMF library routines to interpolate from the native -# forecast model grid to the user-specified output grid (which is defined -# in the model configuration file MODEL_CONFIG_FN in the forecast's run -# directory). -# -#----------------------------------------------------------------------- -# -QUILTING: "TRUE" -PRINT_ESMF: "FALSE" - -WRTCMP_write_groups: "1" -WRTCMP_write_tasks_per_group: "20" - -WRTCMP_output_grid: "''" -WRTCMP_cen_lon: "" -WRTCMP_cen_lat: "" -WRTCMP_lon_lwr_left: "" -WRTCMP_lat_lwr_left: "" -# -# The following are used only for the case of WRTCMP_output_grid set to -# "'rotated_latlon'". -# -WRTCMP_lon_upr_rght: "" -WRTCMP_lat_upr_rght: "" -WRTCMP_dlon: "" -WRTCMP_dlat: "" -# -# The following are used only for the case of WRTCMP_output_grid set to -# "'lambert_conformal'". -# -WRTCMP_stdlat1: "" -WRTCMP_stdlat2: "" -WRTCMP_nx: "" -WRTCMP_ny: "" -WRTCMP_dx: "" -WRTCMP_dy: "" -# -#----------------------------------------------------------------------- -# -# Set PREDEF_GRID_NAME. This parameter specifies a predefined regional -# grid, as follows: -# -# * If PREDEF_GRID_NAME is set to a valid predefined grid name, the grid -# generation method GRID_GEN_METHOD, the (native) grid parameters, and -# the write-component grid parameters are set to predefined values for -# the specified grid, overwriting any settings of these parameters in -# the user-specified experiment configuration file. In addition, if -# the time step DT_ATMOS and the computational parameters LAYOUT_X, -# LAYOUT_Y, and BLOCKSIZE are not specified in that configuration file, -# they are also set to predefined values for the specified grid. -# -# * If PREDEF_GRID_NAME is set to an empty string, it implies the user -# is providing the native grid parameters in the user-specified -# experiment configuration file (EXPT_CONFIG_FN). In this case, the -# grid generation method GRID_GEN_METHOD, the native grid parameters, -# and the write-component grid parameters as well as the time step -# forecast model's main time step DT_ATMOS and the computational -# parameters LAYOUT_X, LAYOUT_Y, and BLOCKSIZE must be set in that -# configuration file; otherwise, the values of all of these parameters -# in this default experiment configuration file will be used. -# -# Setting PREDEF_GRID_NAME provides a convenient method of specifying a -# commonly used set of grid-dependent parameters. The predefined grid -# parameters are specified in the script -# -# $HOMErrfs/ush/set_predef_grid_params.sh -# -#----------------------------------------------------------------------- -# -PREDEF_GRID_NAME: "" -# -#----------------------------------------------------------------------- -# -# Set PREEXISTING_DIR_METHOD. This variable determines the method to use -# use to deal with preexisting directories [e.g ones generated by previous -# calls to the experiment generation script using the same experiment name -# (EXPT_SUBDIR) as the current experiment]. This variable must be set to -# one of "delete", "rename", and "quit". The resulting behavior for each -# of these values is as follows: -# -# * "delete": -# The preexisting directory is deleted and a new directory (having the -# same name as the original preexisting directory) is created. -# -# * "rename": -# The preexisting directory is renamed and a new directory (having the -# same name as the original preexisting directory) is created. The new -# name of the preexisting directory consists of its original name and -# the suffix "_oldNNN", where NNN is a 3-digit integer chosen to make -# the new name unique. -# -# * "quit": -# The preexisting directory is left unchanged, but execution of the -# currently running script is terminated. In this case, the preexisting -# directory must be dealt with manually before rerunning the script. -# -#----------------------------------------------------------------------- -# -PREEXISTING_DIR_METHOD: "delete" -# -#----------------------------------------------------------------------- -# -# Set flags for more detailed messages. Defintitions: -# -# VERBOSE: -# This is a flag that determines whether or not the experiment generation -# and workflow task scripts tend to print out more informational messages. -# -# DEBUG: -# This is a flag that determines whether or not very detailed debugging -# messages are printed to out. Note that if DEBUG is set to TRUE, then -# VERBOSE will also get reset to TRUE if it isn't already. -# -#----------------------------------------------------------------------- -# -VERBOSE: "TRUE" -DEBUG: "FALSE" -# -#----------------------------------------------------------------------- -# -# Set the names of the various rocoto workflow tasks. -# -#----------------------------------------------------------------------- -# -MAKE_GRID_TN: "make_grid" -MAKE_OROG_TN: "make_orog" -MAKE_SFC_CLIMO_TN: "make_sfc_climo" -GET_EXTRN_ICS_TN: "get_extrn_ics" -GET_EXTRN_LBCS_TN: "get_extrn_lbcs" -MAKE_ICS_TN: "make_ics" -MAKE_LBCS_TN: "make_lbcs" -RUN_FCST_TN: "run_fcst" -RUN_POST_TN: "run_post" -GET_OBS: "get_obs" -GET_OBS_CCPA_TN: "get_obs_ccpa" -GET_OBS_MRMS_TN: "get_obs_mrms" -GET_OBS_NDAS_TN: "get_obs_ndas" -VX_TN: "run_vx" -VX_GRIDSTAT_TN: "run_gridstatvx" -VX_GRIDSTAT_REFC_TN: "run_gridstatvx_refc" -VX_GRIDSTAT_RETOP_TN: "run_gridstatvx_retop" -VX_GRIDSTAT_03h_TN: "run_gridstatvx_03h" -VX_GRIDSTAT_06h_TN: "run_gridstatvx_06h" -VX_GRIDSTAT_24h_TN: "run_gridstatvx_24h" -VX_POINTSTAT_TN: "run_pointstatvx" -VX_ENSGRID_TN: "run_ensgridvx" -VX_ENSGRID_03h_TN: "run_ensgridvx_03h" -VX_ENSGRID_06h_TN: "run_ensgridvx_06h" -VX_ENSGRID_24h_TN: "run_ensgridvx_24h" -VX_ENSGRID_REFC_TN: "run_ensgridvx_refc" -VX_ENSGRID_RETOP_TN: "run_ensgridvx_retop" -VX_ENSGRID_MEAN_TN: "run_ensgridvx_mean" -VX_ENSGRID_PROB_TN: "run_ensgridvx_prob" -VX_ENSGRID_MEAN_03h_TN: "run_ensgridvx_mean_03h" -VX_ENSGRID_PROB_03h_TN: "run_ensgridvx_prob_03h" -VX_ENSGRID_MEAN_06h_TN: "run_ensgridvx_mean_06h" -VX_ENSGRID_PROB_06h_TN: "run_ensgridvx_prob_06h" -VX_ENSGRID_MEAN_24h_TN: "run_ensgridvx_mean_24h" -VX_ENSGRID_PROB_24h_TN: "run_ensgridvx_prob_24h" -VX_ENSGRID_PROB_REFC_TN: "run_ensgridvx_prob_refc" -VX_ENSGRID_PROB_RETOP_TN: "run_ensgridvx_prob_retop" -VX_ENSPOINT_TN: "run_enspointvx" -VX_ENSPOINT_MEAN_TN: "run_enspointvx_mean" -VX_ENSPOINT_PROB_TN: "run_enspointvx_prob" -# -#----------------------------------------------------------------------- -# -# Set flags (and related directories) that determine whether various -# workflow tasks should be run. Note that the MAKE_GRID_TN, MAKE_OROG_TN, -# and MAKE_SFC_CLIMO_TN are all cycle-independent tasks, i.e. if they -# are to be run, they do so only once at the beginning of the workflow -# before any cycles are run. Definitions: -# -# RUN_TASK_MAKE_GRID: -# Flag that determines whether the MAKE_GRID_TN task is to be run. If -# this is set to "TRUE", the grid generation task is run and new grid -# files are generated. If it is set to "FALSE", then the scripts look -# for pregenerated grid files in the directory specified by GRID_DIR -# (see below). -# -# GRID_DIR: -# The directory in which to look for pregenerated grid files if -# RUN_TASK_MAKE_GRID is set to "FALSE". -# -# RUN_TASK_MAKE_OROG: -# Same as RUN_TASK_MAKE_GRID but for the MAKE_OROG_TN task. -# -# OROG_DIR: -# Same as GRID_DIR but for the MAKE_OROG_TN task. -# -# RUN_TASK_MAKE_SFC_CLIMO: -# Same as RUN_TASK_MAKE_GRID but for the MAKE_SFC_CLIMO_TN task. -# -# SFC_CLIMO_DIR: -# Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. -# -# DOMAIN_PREGEN_BASEDIR: -# The base directory containing pregenerated grid, orography, and surface -# climatology files. This is an alternative for setting GRID_DIR, -# OROG_DIR, and SFC_CLIMO_DIR individually -# -# For the pregenerated grid specified by PREDEF_GRID_NAME, -# these "fixed" files are located in: -# -# ${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME} -# -# The workflow scripts will create a symlink in the experiment directory -# that will point to a subdirectory (having the name of the grid being -# used) under this directory. This variable should be set to a null -# string in this file, but it can be specified in the user-specified -# workflow configuration file (EXPT_CONFIG_FN). -# -# RUN_TASK_GET_EXTRN_ICS: -# Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. -# -# RUN_TASK_GET_EXTRN_LBCS: -# Flag that determines whether the GET_EXTRN_LBCS_TN task is to be run. -# -# RUN_TASK_MAKE_ICS: -# Flag that determines whether the MAKE_ICS_TN task is to be run. -# -# RUN_TASK_MAKE_LBCS: -# Flag that determines whether the MAKE_LBCS_TN task is to be run. -# -# RUN_TASK_RUN_FCST: -# Flag that determines whether the RUN_FCST_TN task is to be run. -# -# RUN_TASK_RUN_POST: -# Flag that determines whether the RUN_POST_TN task is to be run. -# -# RUN_TASK_VX_GRIDSTAT: -# Flag that determines whether the grid-stat verification task is to be -# run. -# -# RUN_TASK_VX_POINTSTAT: -# Flag that determines whether the point-stat verification task is to be -# run. -# -# RUN_TASK_VX_ENSGRID: -# Flag that determines whether the ensemble-stat verification for gridded -# data task is to be run. -# -# RUN_TASK_VX_ENSPOINT: -# Flag that determines whether the ensemble point verification task is -# to be run. If this flag is set, both ensemble-stat point verification -# and point verification of ensemble-stat output is computed. -# -#----------------------------------------------------------------------- -# -RUN_TASK_MAKE_GRID: "TRUE" -GRID_DIR: "/path/to/pregenerated/grid/files" - -RUN_TASK_MAKE_OROG: "TRUE" -OROG_DIR: "/path/to/pregenerated/orog/files" - -RUN_TASK_MAKE_SFC_CLIMO: "TRUE" -SFC_CLIMO_DIR: "/path/to/pregenerated/surface/climo/files" - -DOMAIN_PREGEN_BASEDIR: "" - -RUN_TASK_GET_EXTRN_ICS: "TRUE" -RUN_TASK_GET_EXTRN_LBCS: "TRUE" -RUN_TASK_MAKE_ICS: "TRUE" -RUN_TASK_MAKE_LBCS: "TRUE" -RUN_TASK_RUN_FCST: "TRUE" -RUN_TASK_RUN_POST: "TRUE" - -RUN_TASK_GET_OBS_CCPA: "FALSE" -RUN_TASK_GET_OBS_MRMS: "FALSE" -RUN_TASK_GET_OBS_NDAS: "FALSE" -RUN_TASK_VX_GRIDSTAT: "FALSE" -RUN_TASK_VX_POINTSTAT: "FALSE" -RUN_TASK_VX_ENSGRID: "FALSE" -RUN_TASK_VX_ENSPOINT: "FALSE" -# -#----------------------------------------------------------------------- -# -# Flag that determines whether MERRA2 aerosol climatology data and -# lookup tables for optics properties are obtained -# -#----------------------------------------------------------------------- -# -USE_MERRA_CLIMO: "FALSE" -# -#----------------------------------------------------------------------- -# -# Set the array parameter containing the names of all the fields that the -# MAKE_SFC_CLIMO_TN task generates on the native FV3-LAM grid. -# -#----------------------------------------------------------------------- -# -SFC_CLIMO_FIELDS: [ -"facsf", -"maximum_snow_albedo", -"slope_type", -"snowfree_albedo", -"soil_type", -"substrate_temperature", -"vegetation_greenness", -"vegetation_type" -] -# -#----------------------------------------------------------------------- -# -# Set parameters associated with the fixed (i.e. static) files. Definitions: -# -# FIXgsm: -# System directory in which the majority of fixed (i.e. time-independent) -# files that are needed to run the FV3-LAM model are located -# -# FIXaer: -# System directory where MERRA2 aerosol climatology files are located -# -# FIXlut: -# System directory where the lookup tables for optics properties are located -# -# TOPO_DIR: -# The location on disk of the static input files used by the make_orog -# task (orog.x and shave.x). Can be the same as FIXgsm. -# -# SFC_CLIMO_INPUT_DIR: -# The location on disk of the static surface climatology input fields, used by -# sfc_climo_gen. These files are only used if RUN_TASK_MAKE_SFC_CLIMO=TRUE -# -# FNGLAC, ..., FNMSKH: -# Names of (some of the) global data files that are assumed to exist in -# a system directory specified (this directory is machine-dependent; -# the experiment generation scripts will set it and store it in the -# variable FIXgsm). These file names also appear directly in the forecast -# model's input namelist file. -# -# FIXgsm_FILES_TO_COPY_TO_FIXam: -# If not running in NCO mode, this array contains the names of the files -# to copy from the FIXgsm system directory to the FIXam directory under -# the experiment directory. Note that the last element has a dummy value. -# This last element will get reset by the workflow generation scripts to -# the name of the ozone production/loss file to copy from FIXgsm. The -# name of this file depends on the ozone parameterization being used, -# and that in turn depends on the CCPP physics suite specified for the -# experiment. Thus, the CCPP physics suite XML must first be read in to -# determine the ozone parameterizaton and then the name of the ozone -# production/loss file. These steps are carried out elsewhere (in one -# of the workflow generation scripts/functions). -# -# FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: -# This array is used to set some of the namelist variables in the forecast -# model's namelist file that represent the relative or absolute paths of -# various fixed files (the first column of the array, where columns are -# delineated by the pipe symbol "|") to the full paths to these files in -# the FIXam directory derived from the corresponding workflow variables -# containing file names (the second column of the array). -# -# FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: -# This array is used to set some of the namelist variables in the forecast -# model's namelist file that represent the relative or absolute paths of -# various fixed files (the first column of the array, where columns are -# delineated by the pipe symbol "|") to the full paths to surface climatology -# files (on the native FV3-LAM grid) in the FIXLAM directory derived from -# the corresponding surface climatology fields (the second column of the -# array). -# -# CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: -# This array specifies the mapping to use between the symlinks that need -# to be created in each cycle directory (these are the "files" that FV3 -# looks for) and their targets in the FIXam directory. The first column -# of the array specifies the symlink to be created, and the second column -# specifies its target file in FIXam (where columns are delineated by the -# pipe symbol "|"). -# -#----------------------------------------------------------------------- -# -# Because the default values are dependent on the platform, we set these -# to a null string which will then be overwritten in setup.sh unless the -# user has specified a different value in config.sh -FIXgsm: "" -FIXaer: "" -FIXlut: "" -TOPO_DIR: "" -SFC_CLIMO_INPUT_DIR: "" - -FNGLAC: &FNGLAC "global_glacier.2x2.grb" -FNMXIC: &FNMXIC "global_maxice.2x2.grb" -FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" -FNSNOC: &FNSNOC "global_snoclim.1.875.grb" -FNZORC: &FNZORC "igbp" -FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" -FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" -FNMSKH: &FNMSKH "seaice_newland.grb" - -FIXgsm_FILES_TO_COPY_TO_FIXam: [ -*FNGLAC, -*FNMXIC, -*FNTSFC, -*FNSNOC, -*FNAISC, -*FNSMCC, -*FNMSKH, -"global_climaeropac_global.txt", -"fix_co2_proj/global_co2historicaldata_2010.txt", -"fix_co2_proj/global_co2historicaldata_2011.txt", -"fix_co2_proj/global_co2historicaldata_2012.txt", -"fix_co2_proj/global_co2historicaldata_2013.txt", -"fix_co2_proj/global_co2historicaldata_2014.txt", -"fix_co2_proj/global_co2historicaldata_2015.txt", -"fix_co2_proj/global_co2historicaldata_2016.txt", -"fix_co2_proj/global_co2historicaldata_2017.txt", -"fix_co2_proj/global_co2historicaldata_2018.txt", -"fix_co2_proj/global_co2historicaldata_2019.txt", -"fix_co2_proj/global_co2historicaldata_2020.txt", -"fix_co2_proj/global_co2historicaldata_2021.txt", -"global_co2historicaldata_glob.txt", -"co2monthlycyc.txt", -"global_h2o_pltc.f77", -"global_hyblev.l65.txt", -"global_zorclim.1x1.grb", -"global_sfc_emissivity_idx.txt", -"global_tg3clim.2.6x1.5.grb", -"global_solarconstant_noaa_an.txt", -"global_albedo4.1x1.grb", -"geo_em.d01.lat-lon.2.5m.HGT_M.nc", -"HGT.Beljaars_filtered.lat-lon.30s_res.nc", -"replace_with_FIXgsm_ozone_prodloss_filename" -] - -# -# It is possible to remove this as a workflow variable and make it only -# a local one since it is used in only one script. -# -FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ -!join_str ["FNGLAC | ",*FNGLAC], -!join_str ["FNMXIC | ",*FNMXIC], -!join_str ["FNTSFC | ",*FNTSFC], -!join_str ["FNSNOC | ",*FNSNOC], -!join_str ["FNAISC | ",*FNAISC], -!join_str ["FNSMCC | ",*FNSMCC], -!join_str ["FNMSKH | ",*FNMSKH] -] -#"FNZORC | $FNZORC", - -FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ -"FNALBC | snowfree_albedo", -"FNALBC2 | facsf", -"FNTG3C | substrate_temperature", -"FNVEGC | vegetation_greenness", -"FNVETC | vegetation_type", -"FNSOTC | soil_type", -"FNVMNC | vegetation_greenness", -"FNVMXC | vegetation_greenness", -"FNSLPC | slope_type", -"FNABSC | maximum_snow_albedo" -] - -CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ -"aerosol.dat | global_climaeropac_global.txt", -"co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", -"co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", -"co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", -"co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", -"co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", -"co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", -"co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", -"co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", -"co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", -"co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", -"co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", -"co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", -"co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", -"co2monthlycyc.txt | co2monthlycyc.txt", -"global_h2oprdlos.f77 | global_h2o_pltc.f77", -"global_albedo4.1x1.grb | global_albedo4.1x1.grb", -"global_zorclim.1x1.grb | global_zorclim.1x1.grb", -"global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", -"sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", -"solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", -"global_o3prdlos.f77 | " -] -# -#----------------------------------------------------------------------- -# -# For each workflow task, set the parameters to pass to the job scheduler -# (e.g. slurm) that will submit a job for each task to be run. These -# parameters include the number of nodes to use to run the job, the MPI -# processes per node, the maximum walltime to allow for the job to complete, -# and the maximum number of times to attempt to run each task. -# -#----------------------------------------------------------------------- -# -# Number of nodes. -# -NNODES_MAKE_GRID: "1" -NNODES_MAKE_OROG: "1" -NNODES_MAKE_SFC_CLIMO: "2" -NNODES_GET_EXTRN_ICS: "1" -NNODES_GET_EXTRN_LBCS: "1" -NNODES_MAKE_ICS: "4" -NNODES_MAKE_LBCS: "4" -NNODES_RUN_FCST: "" # This is calculated in the workflow generation scripts, so no need to set here. -NNODES_RUN_POST: "2" -NNODES_GET_OBS_CCPA: "1" -NNODES_GET_OBS_MRMS: "1" -NNODES_GET_OBS_NDAS: "1" -NNODES_VX_GRIDSTAT: "1" -NNODES_VX_POINTSTAT: "1" -NNODES_VX_ENSGRID: "1" -NNODES_VX_ENSGRID_MEAN: "1" -NNODES_VX_ENSGRID_PROB: "1" -NNODES_VX_ENSPOINT: "1" -NNODES_VX_ENSPOINT_MEAN: "1" -NNODES_VX_ENSPOINT_PROB: "1" -# -# Number of MPI processes per node. -# -PPN_MAKE_GRID: "24" -PPN_MAKE_OROG: "24" -PPN_MAKE_SFC_CLIMO: "24" -PPN_GET_EXTRN_ICS: "1" -PPN_GET_EXTRN_LBCS: "1" -PPN_MAKE_ICS: "12" -PPN_MAKE_LBCS: "12" -PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.sh -PPN_RUN_POST: "24" -PPN_GET_OBS_CCPA: "1" -PPN_GET_OBS_MRMS: "1" -PPN_GET_OBS_NDAS: "1" -PPN_VX_GRIDSTAT: "1" -PPN_VX_POINTSTAT: "1" -PPN_VX_ENSGRID: "1" -PPN_VX_ENSGRID_MEAN: "1" -PPN_VX_ENSGRID_PROB: "1" -PPN_VX_ENSPOINT: "1" -PPN_VX_ENSPOINT_MEAN: "1" -PPN_VX_ENSPOINT_PROB: "1" -# -# Walltimes. -# -WTIME_MAKE_GRID: "00:20:00" -WTIME_MAKE_OROG: "01:00:00" -WTIME_MAKE_SFC_CLIMO: "00:20:00" -WTIME_GET_EXTRN_ICS: "00:45:00" -WTIME_GET_EXTRN_LBCS: "00:45:00" -WTIME_MAKE_ICS: "00:30:00" -WTIME_MAKE_LBCS: "00:30:00" -WTIME_RUN_FCST: "04:30:00" -WTIME_RUN_POST: "00:15:00" -WTIME_GET_OBS_CCPA: "00:45:00" -WTIME_GET_OBS_MRMS: "00:45:00" -WTIME_GET_OBS_NDAS: "02:00:00" -WTIME_VX_GRIDSTAT: "02:00:00" -WTIME_VX_POINTSTAT: "01:00:00" -WTIME_VX_ENSGRID: "01:00:00" -WTIME_VX_ENSGRID_MEAN: "01:00:00" -WTIME_VX_ENSGRID_PROB: "01:00:00" -WTIME_VX_ENSPOINT: "01:00:00" -WTIME_VX_ENSPOINT_MEAN: "01:00:00" -WTIME_VX_ENSPOINT_PROB: "01:00:00" -# -# Maximum number of attempts. -# -MAXTRIES_MAKE_GRID: "2" -MAXTRIES_MAKE_OROG: "2" -MAXTRIES_MAKE_SFC_CLIMO: "2" -MAXTRIES_GET_EXTRN_ICS: "1" -MAXTRIES_GET_EXTRN_LBCS: "1" -MAXTRIES_MAKE_ICS: "1" -MAXTRIES_MAKE_LBCS: "1" -MAXTRIES_RUN_FCST: "1" -MAXTRIES_RUN_POST: "2" -MAXTRIES_GET_OBS_CCPA: "1" -MAXTRIES_GET_OBS_MRMS: "1" -MAXTRIES_GET_OBS_NDAS: "1" -MAXTRIES_VX_GRIDSTAT: "1" -MAXTRIES_VX_GRIDSTAT_REFC: "1" -MAXTRIES_VX_GRIDSTAT_RETOP: "1" -MAXTRIES_VX_GRIDSTAT_03h: "1" -MAXTRIES_VX_GRIDSTAT_06h: "1" -MAXTRIES_VX_GRIDSTAT_24h: "1" -MAXTRIES_VX_POINTSTAT: "1" -MAXTRIES_VX_ENSGRID: "1" -MAXTRIES_VX_ENSGRID_REFC: "1" -MAXTRIES_VX_ENSGRID_RETOP: "1" -MAXTRIES_VX_ENSGRID_03h: "1" -MAXTRIES_VX_ENSGRID_06h: "1" -MAXTRIES_VX_ENSGRID_24h: "1" -MAXTRIES_VX_ENSGRID_MEAN: "1" -MAXTRIES_VX_ENSGRID_PROB: "1" -MAXTRIES_VX_ENSGRID_MEAN_03h: "1" -MAXTRIES_VX_ENSGRID_PROB_03h: "1" -MAXTRIES_VX_ENSGRID_MEAN_06h: "1" -MAXTRIES_VX_ENSGRID_PROB_06h: "1" -MAXTRIES_VX_ENSGRID_MEAN_24h: "1" -MAXTRIES_VX_ENSGRID_PROB_24h: "1" -MAXTRIES_VX_ENSGRID_PROB_REFC: "1" -MAXTRIES_VX_ENSGRID_PROB_RETOP: "1" -MAXTRIES_VX_ENSPOINT: "1" -MAXTRIES_VX_ENSPOINT_MEAN: "1" -MAXTRIES_VX_ENSPOINT_PROB: "1" - -# -#----------------------------------------------------------------------- -# -# Allows an extra parameter to be passed to slurm via XML Native -# command -# -SLURM_NATIVE_CMD: "" -# -#----------------------------------------------------------------------- -# -# Set parameters associated with subhourly forecast model output and -# post-processing. -# -# SUB_HOURLY_POST: -# Flag that indicates whether the forecast model will generate output -# files on a sub-hourly time interval (e.g. 10 minutes, 15 minutes, etc). -# This will also cause the post-processor to process these sub-hourly -# files. If ths is set to "TRUE", then DT_SUBHOURLY_POST_MNTS should be -# set to a value between "00" and "59". -# -# DT_SUB_HOURLY_POST_MNTS: -# Time interval in minutes between the forecast model output files. If -# SUB_HOURLY_POST is set to "TRUE", this needs to be set to a two-digit -# integer between "01" and "59". This is not used if SUB_HOURLY_POST is -# not set to "TRUE". Note that if SUB_HOURLY_POST is set to "TRUE" but -# DT_SUB_HOURLY_POST_MNTS is set to "00", SUB_HOURLY_POST will get reset -# to "FALSE" in the experiment generation scripts (there will be an -# informational message in the log file to emphasize this). -# -#----------------------------------------------------------------------- -# -SUB_HOURLY_POST: "FALSE" -DT_SUBHOURLY_POST_MNTS: "00" -# -#----------------------------------------------------------------------- -# -# Set parameters associated with defining a customized post configuration -# file. -# -# USE_CUSTOM_POST_CONFIG_FILE: -# Flag that determines whether a user-provided custom configuration file -# should be used for post-processing the model data. If this is set to -# "TRUE", then the workflow will use the custom post-processing (UPP) -# configuration file specified in CUSTOM_POST_CONFIG_FP. Otherwise, a -# default configuration file provided in the UPP repository will be -# used. -# -# CUSTOM_POST_CONFIG_FP: -# The full path to the custom post flat file, including filename, to be -# used for post-processing. This is only used if CUSTOM_POST_CONFIG_FILE -# is set to "TRUE". -# -#----------------------------------------------------------------------- -# -USE_CUSTOM_POST_CONFIG_FILE: "FALSE" -CUSTOM_POST_CONFIG_FP: "" -# -#----------------------------------------------------------------------- -# -# Set parameters associated with outputting satellite fields in the UPP -# grib2 files using the Community Radiative Transfer Model (CRTM). -# -# USE_CRTM: -# Flag that defines whether external CRTM coefficient files have been -# staged by the user in order to output synthetic statellite products -# available within the UPP. If this is set to "TRUE", then the workflow -# will check for these files in the directory CRTM_DIR. Otherwise, it is -# assumed that no satellite fields are being requested in the UPP -# configuration. -# -# CRTM_DIR: -# This is the path to the top CRTM fix file directory. This is only used -# if USE_CRTM is set to "TRUE". -# -#----------------------------------------------------------------------- -# -USE_CRTM: "FALSE" -CRTM_DIR: "" -# -#----------------------------------------------------------------------- -# -# Set parameters associated with running ensembles. Definitions: -# -# DO_ENSEMBLE: -# Flag that determines whether to run a set of ensemble forecasts (for -# each set of specified cycles). If this is set to "TRUE", NUM_ENS_MEMBERS -# forecasts are run for each cycle, each with a different set of stochastic -# seed values. Otherwise, a single forecast is run for each cycle. -# -# NUM_ENS_MEMBERS: -# The number of ensemble members to run if DO_ENSEMBLE is set to "TRUE". -# This variable also controls the naming of the ensemble member directories. -# For example, if this is set to "8", the member directories will be named -# mem1, mem2, ..., mem8. If it is set to "08" (note the leading zero), -# the member directories will be named mem01, mem02, ..., mem08. Note, -# however, that after reading in the number of characters in this string -# (in order to determine how many leading zeros, if any, should be placed -# in the names of the member directories), the workflow generation scripts -# strip away those leading zeros. Thus, in the variable definitions file -# (GLOBAL_VAR_DEFNS_FN), this variable appear with its leading zeros -# stripped. This variable is not used if DO_ENSEMBLE is not set to "TRUE". -# -#----------------------------------------------------------------------- -# -DO_ENSEMBLE: "FALSE" -NUM_ENS_MEMBERS: "1" -# -#----------------------------------------------------------------------- -# -# Set default ad-hoc stochastic physics options. -# For detailed documentation of these parameters, see: -# https://stochastic-physics.readthedocs.io/en/ufs_public_release/namelist_options.html -# -#----------------------------------------------------------------------- -# -DO_SHUM: "FALSE" -DO_SPPT: "FALSE" -DO_SKEB: "FALSE" -ISEED_SPPT: "1" -ISEED_SHUM: "2" -ISEED_SKEB: "3" -NEW_LSCALE: "TRUE" -SHUM_MAG: "0.006" #Variable "shum" in input.nml -SHUM_LSCALE: "150000" -SHUM_TSCALE: "21600" #Variable "shum_tau" in input.nml -SHUM_INT: "3600" #Variable "shumint" in input.nml -SPPT_MAG: "0.7" #Variable "sppt" in input.nml -SPPT_LOGIT: "TRUE" -SPPT_LSCALE: "150000" -SPPT_TSCALE: "21600" #Variable "sppt_tau" in input.nml -SPPT_INT: "3600" #Variable "spptint" in input.nml -SPPT_SFCLIMIT: "TRUE" -SKEB_MAG: "0.5" #Variable "skeb" in input.nml -SKEB_LSCALE: "150000" -SKEB_TSCALE: "21600" #Variable "skeb_tau" in input.nml -SKEB_INT: "3600" #Variable "skebint" in input.nml -SKEBNORM: "1" -SKEB_VDOF: "10" -USE_ZMTNBLCK: "FALSE" -# -#----------------------------------------------------------------------- -# -# Set default SPP stochastic physics options. Each SPP option is an array, -# applicable (in order) to the scheme/parameter listed in SPP_VAR_LIST. -# Enter each value of the array in config.sh as shown below without commas -# or single quotes (e.g., SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ). -# Both commas and single quotes will be added by Jinja when creating the -# namelist. -# -# Note that SPP is currently only available for specific physics schemes -# used in the RAP/HRRR physics suite. Users need to be aware of which SDF -# is chosen when turning this option on. -# -# Patterns evolve and are applied at each time step. -# -#----------------------------------------------------------------------- -# -DO_SPP: "FALSE" -SPP_VAR_LIST: [ "pbl", "sfc", "mp", "rad", "gwd" ] -SPP_MAG_LIST: [ "0.2", "0.2", "0.75", "0.2", "0.2" ] #Variable "spp_prt_list" in input.nml -SPP_LSCALE: [ "150000.0", "150000.0", "150000.0", "150000.0", "150000.0" ] -SPP_TSCALE: [ "21600.0", "21600.0", "21600.0", "21600.0", "21600.0" ] #Variable "spp_tau" in input.nml -SPP_SIGTOP1: [ "0.1", "0.1", "0.1", "0.1", "0.1" ] -SPP_SIGTOP2: [ "0.025", "0.025", "0.025", "0.025", "0.025" ] -SPP_STDDEV_CUTOFF: [ "1.5", "1.5", "2.5", "1.5", "1.5" ] -ISEED_SPP: [ "4", "4", "4", "4", "4" ] -# -#----------------------------------------------------------------------- -# -# Turn on SPP in Noah or RUC LSM (support for Noah MP is in progress). -# Please be aware of the SDF that you choose if you wish to turn on LSM -# SPP. -# -# SPP in LSM schemes is handled in the &nam_sfcperts namelist block -# instead of in &nam_sppperts, where all other SPP is implemented. -# -# Perturbations to soil moisture content (SMC) are only applied at the -# first time step. -# -# LSM perturbations include SMC - soil moisture content (volume fraction), -# VGF - vegetation fraction, ALB - albedo, SAL - salinity, -# EMI - emissivity, ZOL - surface roughness (cm), and STC - soil temperature. -# -# Only five perturbations at a time can be applied currently, but all seven -# are shown below. In addition, only one unique iseed value is allowed -# at the moment, and is used for each pattern. -# -DO_LSM_SPP: "FALSE" #If true, sets lndp_type=2, lndp_model_type=2 -LSM_SPP_TSCALE: [ "21600", "21600", "21600", "21600", "21600", "21600", "21600" ] -LSM_SPP_LSCALE: [ "150000", "150000", "150000", "150000", "150000", "150000", "150000" ] -ISEED_LSM_SPP: [ "9" ] -LSM_SPP_VAR_LIST: [ "smc", "vgf", "alb", "sal", "emi", "zol", "stc" ] -LSM_SPP_MAG_LIST: [ "0.017", "0.001", "0.001", "0.001", "0.001", "0.001", "0.2" ] -# -#----------------------------------------------------------------------- -# -# HALO_BLEND: -# Number of rows into the computational domain that should be blended -# with the LBCs. To shut halo blending off, this can be set to zero. -# -#----------------------------------------------------------------------- -# -HALO_BLEND: "10" -# -#----------------------------------------------------------------------- -# -# USE_FVCOM: -# Flag set to update surface conditions in FV3-LAM with fields generated -# from the Finite Volume Community Ocean Model (FVCOM). This will -# replace lake/sea surface temperature, ice surface temperature, and ice -# placement. FVCOM data must already be interpolated to the desired -# FV3-LAM grid. This flag will be used in make_ics to modify sfc_data.nc -# after chgres_cube is run by running the routine process_FVCOM.exe -# -# FVCOM_WCSTART: -# Define if this is a "warm" start or a "cold" start. Setting this to -# "warm" will read in sfc_data.nc generated in a RESTART directory. -# Setting this to "cold" will read in the sfc_data.nc generated from -# chgres_cube in the make_ics portion of the workflow. -# -# FVCOM_DIR: -# User defined directory where FVCOM data already interpolated to FV3-LAM -# grid is located. File name in this path should be "fvcom.nc" to allow -# -# FVCOM_FILE: -# Name of file located in FVCOM_DIR that has FVCOM data interpolated to -# FV3-LAM grid. This file will be copied later to a new location and name -# changed to fvcom.nc -# -#------------------------------------------------------------------------ -# -USE_FVCOM: "FALSE" -FVCOM_WCSTART: "cold" -FVCOM_DIR: "/user/defined/dir/to/fvcom/data" -FVCOM_FILE: "fvcom.nc" -# -#----------------------------------------------------------------------- -# -# COMPILER: -# Type of compiler invoked during the build step. -# -#------------------------------------------------------------------------ -# -COMPILER: "intel" -# -#----------------------------------------------------------------------- -# -# KMP_AFFINITY_*: -# From Intel: "The Intel® runtime library has the ability to bind OpenMP -# threads to physical processing units. The interface is controlled using -# the KMP_AFFINITY environment variable. Depending on the system (machine) -# topology, application, and operating system, thread affinity can have a -# dramatic effect on the application speed. -# -# Thread affinity restricts execution of certain threads (virtual execution -# units) to a subset of the physical processing units in a multiprocessor -# computer. Depending upon the topology of the machine, thread affinity can -# have a dramatic effect on the execution speed of a program." -# -# For more information, see the following link: -# https://software.intel.com/content/www/us/en/develop/documentation/cpp- -# compiler-developer-guide-and-reference/top/optimization-and-programming- -# guide/openmp-support/openmp-library-support/thread-affinity-interface- -# linux-and-windows.html -# -# OMP_NUM_THREADS_*: -# The number of OpenMP threads to use for parallel regions. -# -# OMP_STACKSIZE_*: -# Controls the size of the stack for threads created by the OpenMP -# implementation. -# -# Note that settings for the make_grid and make_orog tasks are not -# included below as they do not use parallelized code. -# -#----------------------------------------------------------------------- -# -KMP_AFFINITY_MAKE_OROG: "disabled" -OMP_NUM_THREADS_MAKE_OROG: "6" -OMP_STACKSIZE_MAKE_OROG: "2048m" - -KMP_AFFINITY_MAKE_SFC_CLIMO: "scatter" -OMP_NUM_THREADS_MAKE_SFC_CLIMO: "1" -OMP_STACKSIZE_MAKE_SFC_CLIMO: "1024m" - -KMP_AFFINITY_MAKE_ICS: "scatter" -OMP_NUM_THREADS_MAKE_ICS: "1" -OMP_STACKSIZE_MAKE_ICS: "1024m" - -KMP_AFFINITY_MAKE_LBCS: "scatter" -OMP_NUM_THREADS_MAKE_LBCS: "1" -OMP_STACKSIZE_MAKE_LBCS: "1024m" - -KMP_AFFINITY_RUN_FCST: "scatter" -OMP_NUM_THREADS_RUN_FCST: "2" # atmos_nthreads in model_configure -OMP_STACKSIZE_RUN_FCST: "1024m" - -KMP_AFFINITY_RUN_POST: "scatter" -OMP_NUM_THREADS_RUN_POST: "1" -OMP_STACKSIZE_RUN_POST: "1024m" -# -#----------------------------------------------------------------------- -# diff --git a/ush/constants.py b/ush/constants.py deleted file mode 100644 index e0ff60a0b0..0000000000 --- a/ush/constants.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/env/bin python3 - -# -#----------------------------------------------------------------------- -# -# Mathematical and physical constants. -# -#----------------------------------------------------------------------- -# - -# Pi. -pi_geom = 3.14159265358979323846264338327 - -# Degrees per radian. -degs_per_radian = 360.0 / (2.0 * pi_geom) - -# Radius of the Earth in meters. -radius_Earth = 6371200.0 - -# -#----------------------------------------------------------------------- -# -# Other. -# -#----------------------------------------------------------------------- -# -valid_vals_BOOLEAN = [True, False] diff --git a/ush/constants.sh b/ush/constants.sh index 26f4cb1b32..f5bce884e8 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -7,13 +7,13 @@ # # Pi. -pi_geom="3.14159265358979323846264338327" +PI_GEOM="3.14159265358979323846264338327" # Degrees per radian. -degs_per_radian=$( bc -l <<< "360.0/(2.0*${pi_geom})" ) +DEGS_PER_RADIAN=$( bc -l <<< "360.0/(2.0*${PI_GEOM})" ) # Radius of the Earth in meters. -radius_Earth="6371200.0" +RADIUS_EARTH="6371200.0" # #----------------------------------------------------------------------- # diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index cd98fcec70..2752d630cc 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import os +import sys +import argparse import unittest from textwrap import dedent from python_utils import import_vars, set_env_var, print_input_args, \ - print_info_msg, print_err_msg_exit, cfg_to_yaml_str + print_info_msg, print_err_msg_exit, cfg_to_yaml_str, \ + load_shell_config from fill_jinja_template import fill_jinja_template @@ -61,6 +64,30 @@ def create_diag_table_file(run_dir): return False return True +def parse_args(argv): + """ Parse command line arguments""" + parser = argparse.ArgumentParser( + description='Creates diagnostic table file.' + ) + + parser.add_argument('-r', '--run-dir', + dest='run_dir', + required=True, + help='Run directory.') + + parser.add_argument('-p', '--path-to-defns', + dest='path_to_defns', + required=True, + help='Path to var_defns file.') + + return parser.parse_args(argv) + +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + cfg = load_shell_config(args.path_to_defns) + import_vars(dictionary=cfg) + create_diag_table_file(args.run_dir) + class Testing(unittest.TestCase): def test_create_diag_table_file(self): path = os.path.join(os.getenv('USHDIR'), "test_data") diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh deleted file mode 100644 index 41ca941bf8..0000000000 --- a/ush/create_diag_table_file.sh +++ /dev/null @@ -1,117 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that creates a diagnostic table file for -# each cycle to be run. -# -#----------------------------------------------------------------------- -# -function create_diag_table_file() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "run_dir" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local diag_table_fp \ - settings -# -#----------------------------------------------------------------------- -# -# Create a diagnostics table file within the specified run directory. -# -#----------------------------------------------------------------------- -# - print_info_msg "$VERBOSE" " -Creating a diagnostics table file (\"${DIAG_TABLE_FN}\") in the specified -run directory... - - run_dir = \"${run_dir}\"" - - diag_table_fp="${run_dir}/${DIAG_TABLE_FN}" - print_info_msg "$VERBOSE" " - -Using the template diagnostics table file: - - diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP} - -to create: - - diag_table_fp = \"${diag_table_fp}\"" - - settings=" -starttime: !datetime ${CDATE} -cres: ${CRES}" - - $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ - print_err_msg_exit " -!!!!!!!!!!!!!!!!! - -fill_jinja_template.py failed! - -!!!!!!!!!!!!!!!!! -" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 9ae1241397..1fafe8a7be 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 import os +import sys +import argparse import unittest from datetime import datetime from textwrap import dedent -from python_utils import import_vars, set_env_var, print_input_args, \ - print_info_msg, print_err_msg_exit, lowercase, cfg_to_yaml_str +from python_utils import import_vars, set_env_var, print_input_args, str_to_type, \ + print_info_msg, print_err_msg_exit, lowercase, cfg_to_yaml_str, \ + load_shell_config from fill_jinja_template import fill_jinja_template @@ -151,7 +154,7 @@ def create_model_configure_file(cdate,run_dir,sub_hourly_post,dt_subhourly_post_ # to, or smaller than one hour. # if sub_hourly_post: - nsout=dt_subhourly_post_mnts*60 / dt_atmos + nsout=(dt_subhourly_post_mnts*60) // dt_atmos output_fh=0 else: output_fh=1 @@ -196,6 +199,55 @@ def create_model_configure_file(cdate,run_dir,sub_hourly_post,dt_subhourly_post_ return True +def parse_args(argv): + """ Parse command line arguments""" + parser = argparse.ArgumentParser( + description='Creates model configuration file.' + ) + + parser.add_argument('-r', '--run-dir', + dest='run_dir', + required=True, + help='Run directory.') + + parser.add_argument('-c', '--cdate', + dest='cdate', + required=True, + help='Date string in YYYYMMDD format.') + + parser.add_argument('-s', '--sub-hourly-post', + dest='sub_hourly_post', + required=True, + help='Set sub hourly post to either TRUE/FALSE by passing corresponding string.') + + parser.add_argument('-d', '--dt-subhourly-post-mnts', + dest='dt_subhourly_post_mnts', + required=True, + help='Subhourly post minitues.') + + parser.add_argument('-t', '--dt-atmos', + dest='dt_atmos', + required=True, + help='Forecast model\'s main time step.') + + parser.add_argument('-p', '--path-to-defns', + dest='path_to_defns', + required=True, + help='Path to var_defns file.') + + return parser.parse_args(argv) + +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + cfg = load_shell_config(args.path_to_defns) + import_vars(dictionary=cfg) + create_model_configure_file( \ + run_dir = args.run_dir, \ + cdate = str_to_type(args.cdate), \ + sub_hourly_post = str_to_type(args.sub_hourly_post), \ + dt_subhourly_post_mnts = str_to_type(args.dt_subhourly_post_mnts), \ + dt_atmos = str_to_type(args.dt_atmos) ) + class Testing(unittest.TestCase): def test_create_model_configure_file(self): path = os.path.join(os.getenv('USHDIR'), "test_data") @@ -240,4 +292,3 @@ def setUp(self): set_env_var('WRTCMP_dx',3000.0) set_env_var('WRTCMP_dy',3000.0) - diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh deleted file mode 100644 index 8141877650..0000000000 --- a/ush/create_model_configure_file.sh +++ /dev/null @@ -1,257 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that creates a model configuration file -# in the specified run directory. -# -#----------------------------------------------------------------------- -# -function create_model_configure_file() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( -cdate \ -run_dir \ -sub_hourly_post \ -dt_subhourly_post_mnts \ -dt_atmos \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local yyyy \ - mm \ - dd \ - hh \ - dot_quilting_dot \ - dot_print_esmf_dot \ - settings \ - model_config_fp -# -#----------------------------------------------------------------------- -# -# Create a model configuration file in the specified run directory. -# -#----------------------------------------------------------------------- -# - print_info_msg "$VERBOSE" " -Creating a model configuration file (\"${MODEL_CONFIG_FN}\") in the specified -run directory (run_dir): - run_dir = \"${run_dir}\"" -# -# Extract from cdate the starting year, month, day, and hour of the forecast. -# - yyyy=${cdate:0:4} - mm=${cdate:4:2} - dd=${cdate:6:2} - hh=${cdate:8:2} -# -# Set parameters in the model configure file. -# - dot_quilting_dot="."$(echo_lowercase $QUILTING)"." - dot_print_esmf_dot="."$(echo_lowercase $PRINT_ESMF)"." - dot_cpl_dot="."$(echo_lowercase $CPL)"." - dot_write_dopost="."$(echo_lowercase $WRITE_DOPOST)"." -# -#----------------------------------------------------------------------- -# -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the jinja variables in the template -# model_configure file should be set to. -# -#----------------------------------------------------------------------- -# - settings="\ - 'PE_MEMBER01': ${PE_MEMBER01} - 'print_esmf': ${dot_print_esmf_dot} - 'start_year': $yyyy - 'start_month': $mm - 'start_day': $dd - 'start_hour': $hh - 'nhours_fcst': ${FCST_LEN_HRS} - 'dt_atmos': ${DT_ATMOS} - 'cpl': ${dot_cpl_dot} - 'atmos_nthreads': ${OMP_NUM_THREADS_RUN_FCST} - 'restart_interval': ${RESTART_INTERVAL} - 'write_dopost': ${dot_write_dopost} - 'quilting': ${dot_quilting_dot} - 'output_grid': ${WRTCMP_output_grid}" -# 'output_grid': \'${WRTCMP_output_grid}\'" -# -# If the write-component is to be used, then specify a set of computational -# parameters and a set of grid parameters. The latter depends on the type -# (coordinate system) of the grid that the write-component will be using. -# - if [ "$QUILTING" = "TRUE" ]; then - - settings="${settings} - 'write_groups': ${WRTCMP_write_groups} - 'write_tasks_per_group': ${WRTCMP_write_tasks_per_group} - 'cen_lon': ${WRTCMP_cen_lon} - 'cen_lat': ${WRTCMP_cen_lat} - 'lon1': ${WRTCMP_lon_lwr_left} - 'lat1': ${WRTCMP_lat_lwr_left}" - - if [ "${WRTCMP_output_grid}" = "lambert_conformal" ]; then - - settings="${settings} - 'stdlat1': ${WRTCMP_stdlat1} - 'stdlat2': ${WRTCMP_stdlat2} - 'nx': ${WRTCMP_nx} - 'ny': ${WRTCMP_ny} - 'dx': ${WRTCMP_dx} - 'dy': ${WRTCMP_dy} - 'lon2': \"\" - 'lat2': \"\" - 'dlon': \"\" - 'dlat': \"\"" - - elif [ "${WRTCMP_output_grid}" = "regional_latlon" ] || \ - [ "${WRTCMP_output_grid}" = "rotated_latlon" ]; then - - settings="${settings} - 'lon2': ${WRTCMP_lon_upr_rght} - 'lat2': ${WRTCMP_lat_upr_rght} - 'dlon': ${WRTCMP_dlon} - 'dlat': ${WRTCMP_dlat} - 'stdlat1': \"\" - 'stdlat2': \"\" - 'nx': \"\" - 'ny': \"\" - 'dx': \"\" - 'dy': \"\"" - - fi - - fi -# -# If sub_hourly_post is set to "TRUE", then the forecast model must be -# directed to generate output files on a sub-hourly interval. Do this -# by specifying the output interval in the model configuration file -# (MODEL_CONFIG_FN) in units of number of forecat model time steps (nsout). -# nsout is calculated using the user-specified output time interval -# dt_subhourly_post_mnts (in units of minutes) and the forecast model's -# main time step dt_atmos (in units of seconds). Note that nsout is -# guaranteed to be an integer because the experiment generation scripts -# require that dt_subhourly_post_mnts (after conversion to seconds) be -# evenly divisible by dt_atmos. Also, in this case, the variable output_fh -# [which specifies the output interval in hours; -# see the jinja model_config template file] is set to 0, although this -# doesn't matter because any positive of nsout will override output_fh. -# -# If sub_hourly_post is set to "FALSE", then the workflow is hard-coded -# (in the jinja model_config template file) to direct the forecast model -# to output files every hour. This is done by setting (1) output_fh to 1 -# here, and (2) nsout to -1 here which turns off output by time step interval. -# -# Note that the approach used here of separating how hourly and subhourly -# output is handled should be changed/generalized/simplified such that -# the user should only need to specify the output time interval (there -# should be no need to specify a flag like sub_hourly_post); the workflow -# should then be able to direct the model to output files with that time -# interval and to direct the post-processor to process those files -# regardless of whether that output time interval is larger than, equal -# to, or smaller than one hour. -# - if [ "${sub_hourly_post}" = "TRUE" ]; then - nsout=$(( dt_subhourly_post_mnts*60 / dt_atmos )) - output_fh=0 - else - output_fh=1 - nsout=-1 - fi - settings="${settings} - 'output_fh': ${output_fh} - 'nsout': ${nsout}" - - print_info_msg $VERBOSE " -The variable \"settings\" specifying values to be used in the \"${MODEL_CONFIG_FN}\" -file has been set as follows: -#----------------------------------------------------------------------- -settings = -$settings" -# -#----------------------------------------------------------------------- -# -# Call a python script to generate the experiment's actual MODEL_CONFIG_FN -# file from the template file. -# -#----------------------------------------------------------------------- -# - model_config_fp="${run_dir}/${MODEL_CONFIG_FN}" - $USHDIR/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${MODEL_CONFIG_TMPL_FP} \ - -o ${model_config_fp} || \ - print_err_msg_exit "\ -Call to python script fill_jinja_template.py to create a \"${MODEL_CONFIG_FN}\" -file from a jinja2 template failed. Parameters passed to this script are: - Full path to template rocoto XML file: - MODEL_CONFIG_TMPL_FP = \"${MODEL_CONFIG_TMPL_FP}\" - Full path to output rocoto XML file: - model_config_fp = \"${model_config_fp}\" - Namelist settings specified on command line: - settings = -$settings" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index ad1e205d45..f21fe308aa 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -15,7 +15,7 @@ from setup import setup from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames -from get_crontab_contents import get_crontab_contents +from get_crontab_contents import add_crontab_line from fill_jinja_template import fill_jinja_template from set_namelist import set_namelist @@ -449,77 +449,7 @@ def generate_FV3LAM_wflow(): #----------------------------------------------------------------------- # if USE_CRON_TO_RELAUNCH: - # - # Make a backup copy of the user's crontab file and save it in a file. - # - time_stamp = datetime.now().strftime("%F_%T") - crontab_backup_fp=os.path.join(EXPTDIR,f"crontab.bak.{time_stamp}") - print_info_msg(f''' - Copying contents of user cron table to backup file: - crontab_backup_fp = \"{crontab_backup_fp}\"''',verbose=VERBOSE) - - global called_from_cron - try: called_from_cron - except: called_from_cron = False - - crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=called_from_cron) - # To create the backup crontab file and add a new job to the user's - # existing cron table, use the "printf" command, not "echo", to print - # out variables. This is because "echo" will add a newline at the end - # of its output even if its input argument is a null string, resulting - # in extranous blank lines in the backup crontab file and/or the cron - # table itself. Using "printf" prevents the appearance of these blank - # lines. - run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''') - # - # Below, we use "grep" to determine whether the crontab line that the - # variable CRONTAB_LINE contains is already present in the cron table. - # For that purpose, we need to escape the asterisks in the string in - # CRONTAB_LINE with backslashes. Do this next. - # - (_,crontab_line_esc_astr,_) = run_command(f'''printf "%s" '{CRONTAB_LINE}' | \ - {SED} -r -e "s%[*]%\\\\*%g"''') - # In the grep command below, the "^" at the beginning of the string - # passed to grep is a start-of-line anchor, and the "$" at the end is - # an end-of-line anchor. Thus, in order for grep to find a match on - # any given line of the cron table's contents, that line must contain - # exactly the string in the variable crontab_line_esc_astr without any - # leading or trailing characters. This is to eliminate situations in - # which a line in the cron table contains the string in crontab_line_esc_astr - # but is precedeeded, for example, by the comment character "#" (in which - # case cron ignores that line) and/or is followed by further commands - # that are not part of the string in crontab_line_esc_astr (in which - # case it does something more than the command portion of the string in - # crontab_line_esc_astr does). - # - if MACHINE == "WCOSS_DELL_P3": - (exit_status,grep_output,_)=run_command(f'''grep '^{crontab_line_esc_astr}$' "/u/{USER}/cron/mycrontab"''') - else: - (exit_status,grep_output,_)=run_command(f'''printf "%s" '{crontab_contents}' | grep "^{crontab_line_esc_astr}$"''') - - if exit_status == 0: - - print_info_msg(f''' - The following line already exists in the cron table and thus will not be - added: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''') - - else: - - print_info_msg(f''' - Adding the following line to the user's cron table in order to automatically - resubmit SRW workflow: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE) - - if MACHINE == "WCOSS_DELL_P3": - run_command(f'''printf "%s\n" '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''') - else: - # Add a newline to the end of crontab_contents only if it is not empty. - # This is needed so that when CRONTAB_LINE is printed out, it appears on - # a separate line. - if crontab_contents: - crontab_contents += "\n" - run_command(f'''( printf "%s" '{crontab_contents}'; printf "%s\n" '{CRONTAB_LINE}' ) | {crontab_cmd}''') + add_crontab_line() # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh deleted file mode 100755 index d9f2299139..0000000000 --- a/ush/generate_FV3LAM_wflow.sh +++ /dev/null @@ -1,1213 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets up a forecast -# experiment and creates a workflow (according to the parameters speci- -# fied in the configuration file; see instructions). -# -#----------------------------------------------------------------------- -# -function generate_FV3LAM_wflow() { -printf "\ -======================================================================== -======================================================================== - -Starting experiment generation... - -======================================================================== -======================================================================== -" -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -if [[ $(uname -s) == Darwin ]]; then - local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) -else - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -fi -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# -ushdir="${scrfunc_dir}" -# -#----------------------------------------------------------------------- -# -# Source bash utility functions and other necessary files. -# -#----------------------------------------------------------------------- -# -. $ushdir/source_util_funcs.sh -. $ushdir/get_crontab_contents.sh -. $ushdir/set_FV3nml_sfc_climo_filenames.sh -# -#----------------------------------------------------------------------- -# -# Run python checks -# -#----------------------------------------------------------------------- -# - -# This line will return two numbers: the python major and minor versions -pyversion=($(/usr/bin/env python3 -c 'import platform; major, minor, patch = platform.python_version_tuple(); print(major); print(minor)')) - -#Now, set an error check variable so that we can print all python errors rather than just the first -pyerrors=0 - -# Check that the call to python3 returned no errors, then check if the -# python3 minor version is 6 or higher -if [[ -z "$pyversion" ]];then - print_info_msg "\ - - Error: python3 not found" - pyerrors=$((pyerrors+1)) -else - if [[ ${#pyversion[@]} -lt 2 ]]; then - print_info_msg "\ - - Error retrieving python3 version" - pyerrors=$((pyerrors+1)) - elif [[ ${pyversion[1]} -lt 6 ]]; then - print_info_msg "\ - - Error: python version must be 3.6 or higher - python version: ${pyversion[*]}" - pyerrors=$((pyerrors+1)) - fi -fi - -#Next, check for the non-standard python packages: jinja2, yaml, and f90nml -pkgs=(jinja2 yaml f90nml) -for pkg in ${pkgs[@]} ; do - if ! /usr/bin/env python3 -c "import ${pkg}" &> /dev/null; then - print_info_msg "\ - - Error: python module ${pkg} not available" - pyerrors=$((pyerrors+1)) - fi -done - -#Finally, check if the number of errors is >0, and if so exit with helpful message -if [ $pyerrors -gt 0 ];then - print_err_msg_exit "\ - Errors found: check your python environment - - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - -" -fi -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Source the file that defines and then calls the setup function. The -# setup function in turn first sources the default configuration file -# (which contains default values for the experiment/workflow parameters) -# and then sources the user-specified configuration file (which contains -# user-specified values for a subset of the experiment/workflow parame- -# ters that override their default values). -# -#----------------------------------------------------------------------- -# -. $ushdir/setup.sh -# -#----------------------------------------------------------------------- -# -# Set the full path to the experiment's rocoto workflow xml file. This -# file will be placed at the top level of the experiment directory and -# then used by rocoto to run the workflow. -# -#----------------------------------------------------------------------- -# -WFLOW_XML_FP="$EXPTDIR/${WFLOW_XML_FN}" -# -#----------------------------------------------------------------------- -# -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the jinja variables in the template rocoto -# XML should be set to. These values are set either in the user-specified -# workflow configuration file (EXPT_CONFIG_FN) or in the setup.sh script -# sourced above. Then call the python script that generates the XML. -# -#----------------------------------------------------------------------- -# -if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then - - template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" - - print_info_msg " -Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML -file (template_xml_fp): - template_xml_fp = \"${template_xml_fp}\" - WFLOW_XML_FP = \"${WFLOW_XML_FP}\"" - - ensmem_indx_name="\"\"" - uscore_ensmem_name="\"\"" - slash_ensmem_subdir="\"\"" - if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - ensmem_indx_name="mem" - uscore_ensmem_name="_mem#${ensmem_indx_name}#" - slash_ensmem_subdir="/mem#${ensmem_indx_name}#" - fi - - settings="\ -# -# Parameters needed by the job scheduler. -# - 'account': $ACCOUNT - 'sched': $SCHED - 'partition_default': ${PARTITION_DEFAULT} - 'queue_default': ${QUEUE_DEFAULT} - 'partition_hpss': ${PARTITION_HPSS} - 'queue_hpss': ${QUEUE_HPSS} - 'partition_fcst': ${PARTITION_FCST} - 'queue_fcst': ${QUEUE_FCST} - 'machine': ${MACHINE} - 'slurm_native_cmd': ${SLURM_NATIVE_CMD} -# -# Workflow task names. -# - 'make_grid_tn': ${MAKE_GRID_TN} - 'make_orog_tn': ${MAKE_OROG_TN} - 'make_sfc_climo_tn': ${MAKE_SFC_CLIMO_TN} - 'get_extrn_ics_tn': ${GET_EXTRN_ICS_TN} - 'get_extrn_lbcs_tn': ${GET_EXTRN_LBCS_TN} - 'make_ics_tn': ${MAKE_ICS_TN} - 'make_lbcs_tn': ${MAKE_LBCS_TN} - 'run_fcst_tn': ${RUN_FCST_TN} - 'run_post_tn': ${RUN_POST_TN} - 'get_obs_ccpa_tn': ${GET_OBS_CCPA_TN} - 'get_obs_ndas_tn': ${GET_OBS_NDAS_TN} - 'get_obs_mrms_tn': ${GET_OBS_MRMS_TN} - 'vx_tn': ${VX_TN} - 'vx_gridstat_tn': ${VX_GRIDSTAT_TN} - 'vx_gridstat_refc_tn': ${VX_GRIDSTAT_REFC_TN} - 'vx_gridstat_retop_tn': ${VX_GRIDSTAT_RETOP_TN} - 'vx_gridstat_03h_tn': ${VX_GRIDSTAT_03h_TN} - 'vx_gridstat_06h_tn': ${VX_GRIDSTAT_06h_TN} - 'vx_gridstat_24h_tn': ${VX_GRIDSTAT_24h_TN} - 'vx_pointstat_tn': ${VX_POINTSTAT_TN} - 'vx_ensgrid_tn': ${VX_ENSGRID_TN} - 'vx_ensgrid_refc_tn': ${VX_ENSGRID_REFC_TN} - 'vx_ensgrid_retop_tn': ${VX_ENSGRID_RETOP_TN} - 'vx_ensgrid_03h_tn': ${VX_ENSGRID_03h_TN} - 'vx_ensgrid_06h_tn': ${VX_ENSGRID_06h_TN} - 'vx_ensgrid_24h_tn': ${VX_ENSGRID_24h_TN} - 'vx_ensgrid_mean_tn': ${VX_ENSGRID_MEAN_TN} - 'vx_ensgrid_prob_tn': ${VX_ENSGRID_PROB_TN} - 'vx_ensgrid_mean_03h_tn': ${VX_ENSGRID_MEAN_03h_TN} - 'vx_ensgrid_prob_03h_tn': ${VX_ENSGRID_PROB_03h_TN} - 'vx_ensgrid_mean_06h_tn': ${VX_ENSGRID_MEAN_06h_TN} - 'vx_ensgrid_prob_06h_tn': ${VX_ENSGRID_PROB_06h_TN} - 'vx_ensgrid_mean_24h_tn': ${VX_ENSGRID_MEAN_24h_TN} - 'vx_ensgrid_prob_24h_tn': ${VX_ENSGRID_PROB_24h_TN} - 'vx_ensgrid_prob_refc_tn': ${VX_ENSGRID_PROB_REFC_TN} - 'vx_ensgrid_prob_retop_tn': ${VX_ENSGRID_PROB_RETOP_TN} - 'vx_enspoint_tn': ${VX_ENSPOINT_TN} - 'vx_enspoint_mean_tn': ${VX_ENSPOINT_MEAN_TN} - 'vx_enspoint_prob_tn': ${VX_ENSPOINT_PROB_TN} -# -# Entity used to load the module file for each GET_OBS_* task. -# - 'get_obs': ${GET_OBS} -# -# Number of nodes to use for each task. -# - 'nnodes_make_grid': ${NNODES_MAKE_GRID} - 'nnodes_make_orog': ${NNODES_MAKE_OROG} - 'nnodes_make_sfc_climo': ${NNODES_MAKE_SFC_CLIMO} - 'nnodes_get_extrn_ics': ${NNODES_GET_EXTRN_ICS} - 'nnodes_get_extrn_lbcs': ${NNODES_GET_EXTRN_LBCS} - 'nnodes_make_ics': ${NNODES_MAKE_ICS} - 'nnodes_make_lbcs': ${NNODES_MAKE_LBCS} - 'nnodes_run_fcst': ${NNODES_RUN_FCST} - 'nnodes_run_post': ${NNODES_RUN_POST} - 'nnodes_get_obs_ccpa': ${NNODES_GET_OBS_CCPA} - 'nnodes_get_obs_mrms': ${NNODES_GET_OBS_MRMS} - 'nnodes_get_obs_ndas': ${NNODES_GET_OBS_NDAS} - 'nnodes_vx_gridstat': ${NNODES_VX_GRIDSTAT} - 'nnodes_vx_pointstat': ${NNODES_VX_POINTSTAT} - 'nnodes_vx_ensgrid': ${NNODES_VX_ENSGRID} - 'nnodes_vx_ensgrid_mean': ${NNODES_VX_ENSGRID_MEAN} - 'nnodes_vx_ensgrid_prob': ${NNODES_VX_ENSGRID_PROB} - 'nnodes_vx_enspoint': ${NNODES_VX_ENSPOINT} - 'nnodes_vx_enspoint_mean': ${NNODES_VX_ENSPOINT_MEAN} - 'nnodes_vx_enspoint_prob': ${NNODES_VX_ENSPOINT_PROB} -# -# Number of cores used for a task -# - 'ncores_run_fcst': ${PE_MEMBER01} - 'native_run_fcst': --cpus-per-task ${OMP_NUM_THREADS_RUN_FCST} --exclusive -# -# Number of logical processes per node for each task. If running without -# threading, this is equal to the number of MPI processes per node. -# - 'ppn_make_grid': ${PPN_MAKE_GRID} - 'ppn_make_orog': ${PPN_MAKE_OROG} - 'ppn_make_sfc_climo': ${PPN_MAKE_SFC_CLIMO} - 'ppn_get_extrn_ics': ${PPN_GET_EXTRN_ICS} - 'ppn_get_extrn_lbcs': ${PPN_GET_EXTRN_LBCS} - 'ppn_make_ics': ${PPN_MAKE_ICS} - 'ppn_make_lbcs': ${PPN_MAKE_LBCS} - 'ppn_run_fcst': ${PPN_RUN_FCST} - 'ppn_run_post': ${PPN_RUN_POST} - 'ppn_get_obs_ccpa': ${PPN_GET_OBS_CCPA} - 'ppn_get_obs_mrms': ${PPN_GET_OBS_MRMS} - 'ppn_get_obs_ndas': ${PPN_GET_OBS_NDAS} - 'ppn_vx_gridstat': ${PPN_VX_GRIDSTAT} - 'ppn_vx_pointstat': ${PPN_VX_POINTSTAT} - 'ppn_vx_ensgrid': ${PPN_VX_ENSGRID} - 'ppn_vx_ensgrid_mean': ${PPN_VX_ENSGRID_MEAN} - 'ppn_vx_ensgrid_prob': ${PPN_VX_ENSGRID_PROB} - 'ppn_vx_enspoint': ${PPN_VX_ENSPOINT} - 'ppn_vx_enspoint_mean': ${PPN_VX_ENSPOINT_MEAN} - 'ppn_vx_enspoint_prob': ${PPN_VX_ENSPOINT_PROB} -# -# Maximum wallclock time for each task. -# - 'wtime_make_grid': ${WTIME_MAKE_GRID} - 'wtime_make_orog': ${WTIME_MAKE_OROG} - 'wtime_make_sfc_climo': ${WTIME_MAKE_SFC_CLIMO} - 'wtime_get_extrn_ics': ${WTIME_GET_EXTRN_ICS} - 'wtime_get_extrn_lbcs': ${WTIME_GET_EXTRN_LBCS} - 'wtime_make_ics': ${WTIME_MAKE_ICS} - 'wtime_make_lbcs': ${WTIME_MAKE_LBCS} - 'wtime_run_fcst': ${WTIME_RUN_FCST} - 'wtime_run_post': ${WTIME_RUN_POST} - 'wtime_get_obs_ccpa': ${WTIME_GET_OBS_CCPA} - 'wtime_get_obs_mrms': ${WTIME_GET_OBS_MRMS} - 'wtime_get_obs_ndas': ${WTIME_GET_OBS_NDAS} - 'wtime_vx_gridstat': ${WTIME_VX_GRIDSTAT} - 'wtime_vx_pointstat': ${WTIME_VX_POINTSTAT} - 'wtime_vx_ensgrid': ${WTIME_VX_ENSGRID} - 'wtime_vx_ensgrid_mean': ${WTIME_VX_ENSGRID_MEAN} - 'wtime_vx_ensgrid_prob': ${WTIME_VX_ENSGRID_PROB} - 'wtime_vx_enspoint': ${WTIME_VX_ENSPOINT} - 'wtime_vx_enspoint_mean': ${WTIME_VX_ENSPOINT_MEAN} - 'wtime_vx_enspoint_prob': ${WTIME_VX_ENSPOINT_PROB} -# -# Maximum number of tries for each task. -# - 'maxtries_make_grid': ${MAXTRIES_MAKE_GRID} - 'maxtries_make_orog': ${MAXTRIES_MAKE_OROG} - 'maxtries_make_sfc_climo': ${MAXTRIES_MAKE_SFC_CLIMO} - 'maxtries_get_extrn_ics': ${MAXTRIES_GET_EXTRN_ICS} - 'maxtries_get_extrn_lbcs': ${MAXTRIES_GET_EXTRN_LBCS} - 'maxtries_make_ics': ${MAXTRIES_MAKE_ICS} - 'maxtries_make_lbcs': ${MAXTRIES_MAKE_LBCS} - 'maxtries_run_fcst': ${MAXTRIES_RUN_FCST} - 'maxtries_run_post': ${MAXTRIES_RUN_POST} - 'maxtries_get_obs_ccpa': ${MAXTRIES_GET_OBS_CCPA} - 'maxtries_get_obs_mrms': ${MAXTRIES_GET_OBS_MRMS} - 'maxtries_get_obs_ndas': ${MAXTRIES_GET_OBS_NDAS} - 'maxtries_vx_gridstat': ${MAXTRIES_VX_GRIDSTAT} - 'maxtries_vx_gridstat_refc': ${MAXTRIES_VX_GRIDSTAT_REFC} - 'maxtries_vx_gridstat_retop': ${MAXTRIES_VX_GRIDSTAT_RETOP} - 'maxtries_vx_gridstat_03h': ${MAXTRIES_VX_GRIDSTAT_03h} - 'maxtries_vx_gridstat_06h': ${MAXTRIES_VX_GRIDSTAT_06h} - 'maxtries_vx_gridstat_24h': ${MAXTRIES_VX_GRIDSTAT_24h} - 'maxtries_vx_pointstat': ${MAXTRIES_VX_POINTSTAT} - 'maxtries_vx_ensgrid': ${MAXTRIES_VX_ENSGRID} - 'maxtries_vx_ensgrid_refc': ${MAXTRIES_VX_ENSGRID_REFC} - 'maxtries_vx_ensgrid_retop': ${MAXTRIES_VX_ENSGRID_RETOP} - 'maxtries_vx_ensgrid_03h': ${MAXTRIES_VX_ENSGRID_03h} - 'maxtries_vx_ensgrid_06h': ${MAXTRIES_VX_ENSGRID_06h} - 'maxtries_vx_ensgrid_24h': ${MAXTRIES_VX_ENSGRID_24h} - 'maxtries_vx_ensgrid_mean': ${MAXTRIES_VX_ENSGRID_MEAN} - 'maxtries_vx_ensgrid_prob': ${MAXTRIES_VX_ENSGRID_PROB} - 'maxtries_vx_ensgrid_mean_03h': ${MAXTRIES_VX_ENSGRID_MEAN_03h} - 'maxtries_vx_ensgrid_prob_03h': ${MAXTRIES_VX_ENSGRID_PROB_03h} - 'maxtries_vx_ensgrid_mean_06h': ${MAXTRIES_VX_ENSGRID_MEAN_06h} - 'maxtries_vx_ensgrid_prob_06h': ${MAXTRIES_VX_ENSGRID_PROB_06h} - 'maxtries_vx_ensgrid_mean_24h': ${MAXTRIES_VX_ENSGRID_MEAN_24h} - 'maxtries_vx_ensgrid_prob_24h': ${MAXTRIES_VX_ENSGRID_PROB_24h} - 'maxtries_vx_ensgrid_prob_refc': ${MAXTRIES_VX_ENSGRID_PROB_REFC} - 'maxtries_vx_ensgrid_prob_retop': ${MAXTRIES_VX_ENSGRID_PROB_RETOP} - 'maxtries_vx_enspoint': ${MAXTRIES_VX_ENSPOINT} - 'maxtries_vx_enspoint_mean': ${MAXTRIES_VX_ENSPOINT_MEAN} - 'maxtries_vx_enspoint_prob': ${MAXTRIES_VX_ENSPOINT_PROB} -# -# Flags that specify whether to run the preprocessing or -# verification-related tasks. -# - 'run_task_make_grid': ${RUN_TASK_MAKE_GRID} - 'run_task_make_orog': ${RUN_TASK_MAKE_OROG} - 'run_task_make_sfc_climo': ${RUN_TASK_MAKE_SFC_CLIMO} - 'run_task_get_extrn_ics': ${RUN_TASK_GET_EXTRN_ICS} - 'run_task_get_extrn_lbcs': ${RUN_TASK_GET_EXTRN_LBCS} - 'run_task_make_ics': ${RUN_TASK_MAKE_ICS} - 'run_task_make_lbcs': ${RUN_TASK_MAKE_LBCS} - 'run_task_run_fcst': ${RUN_TASK_RUN_FCST} - 'run_task_run_post': ${RUN_TASK_RUN_POST} - 'run_task_get_obs_ccpa': ${RUN_TASK_GET_OBS_CCPA} - 'run_task_get_obs_mrms': ${RUN_TASK_GET_OBS_MRMS} - 'run_task_get_obs_ndas': ${RUN_TASK_GET_OBS_NDAS} - 'run_task_vx_gridstat': ${RUN_TASK_VX_GRIDSTAT} - 'run_task_vx_pointstat': ${RUN_TASK_VX_POINTSTAT} - 'run_task_vx_ensgrid': ${RUN_TASK_VX_ENSGRID} - 'run_task_vx_enspoint': ${RUN_TASK_VX_ENSPOINT} -# -# Number of physical cores per node for the current machine. -# - 'ncores_per_node': ${NCORES_PER_NODE} -# -# Directories and files. -# - 'jobsdir': $JOBSDIR - 'logdir': $LOGDIR - 'scriptsdir': $SCRIPTSDIR - 'cycle_basedir': ${CYCLE_BASEDIR} - 'global_var_defns_fp': ${GLOBAL_VAR_DEFNS_FP} - 'load_modules_run_task_fp': ${LOAD_MODULES_RUN_TASK_FP} -# -# External model information for generating ICs and LBCs. -# - 'extrn_mdl_name_ics': ${EXTRN_MDL_NAME_ICS} - 'extrn_mdl_name_lbcs': ${EXTRN_MDL_NAME_LBCS} -# -# Parameters that determine the set of cycles to run. -# - 'date_first_cycl': ${DATE_FIRST_CYCL} - 'date_last_cycl': ${DATE_LAST_CYCL} - 'cdate_first_cycl': !datetime ${DATE_FIRST_CYCL}${CYCL_HRS[0]} - 'cycl_hrs': [ $( printf "\'%s\', " "${CYCL_HRS[@]}" ) ] - 'cycl_freq': !!str ${INCR_CYCL_FREQ}:00:00 -# -# Forecast length (same for all cycles). -# - 'fcst_len_hrs': ${FCST_LEN_HRS} -# -# Inline post -# - 'write_dopost': ${WRITE_DOPOST} -# -# METPlus-specific information -# - 'model': ${MODEL} - 'met_install_dir': ${MET_INSTALL_DIR} - 'met_bin_exec': ${MET_BIN_EXEC} - 'metplus_path': ${METPLUS_PATH} - 'vx_config_dir': ${VX_CONFIG_DIR} - 'metplus_conf': ${METPLUS_CONF} - 'met_config': ${MET_CONFIG} - 'ccpa_obs_dir': ${CCPA_OBS_DIR} - 'mrms_obs_dir': ${MRMS_OBS_DIR} - 'ndas_obs_dir': ${NDAS_OBS_DIR} -# -# Ensemble-related parameters. -# - 'do_ensemble': ${DO_ENSEMBLE} - 'num_ens_members': ${NUM_ENS_MEMBERS} - 'ndigits_ensmem_names': !!str ${NDIGITS_ENSMEM_NAMES} - 'ensmem_indx_name': ${ensmem_indx_name} - 'uscore_ensmem_name': ${uscore_ensmem_name} - 'slash_ensmem_subdir': ${slash_ensmem_subdir} -# -# Parameters associated with subhourly post-processed output -# - 'sub_hourly_post': ${SUB_HOURLY_POST} - 'delta_min': ${DT_SUBHOURLY_POST_MNTS} - 'first_fv3_file_tstr': "000:"`$DATE_UTIL -d "${DATE_FIRST_CYCL} +${DT_ATMOS} seconds" +%M:%S` -" # End of "settings" variable. - - print_info_msg "$VERBOSE" " -The variable \"settings\" specifying values of the rococo XML variables -has been set as follows: -#----------------------------------------------------------------------- -settings = -$settings" - -# -# Call the python script to generate the experiment's actual XML file -# from the jinja template file. -# - $USHDIR/fill_jinja_template.py -q \ - -u "${settings}" \ - -t ${template_xml_fp} \ - -o ${WFLOW_XML_FP} || \ - print_err_msg_exit "\ -Call to python script fill_jinja_template.py to create a rocoto workflow -XML file from a template file failed. Parameters passed to this script -are: - Full path to template rocoto XML file: - template_xml_fp = \"${template_xml_fp}\" - Full path to output rocoto XML file: - WFLOW_XML_FP = \"${WFLOW_XML_FP}\" - Namelist settings specified on command line: - settings = -$settings" - -fi -# -#----------------------------------------------------------------------- -# -# Create a symlink in the experiment directory that points to the workflow -# (re)launch script. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Creating symlink in the experiment directory (EXPTDIR) that points to the -workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): - EXPTDIR = \"${EXPTDIR}\" - WFLOW_LAUNCH_SCRIPT_FP = \"${WFLOW_LAUNCH_SCRIPT_FP}\"" -create_symlink_to_file target="${WFLOW_LAUNCH_SCRIPT_FP}" \ - symlink="${EXPTDIR}/${WFLOW_LAUNCH_SCRIPT_FN}" \ - relative="FALSE" -# -#----------------------------------------------------------------------- -# -# If USE_CRON_TO_RELAUNCH is set to TRUE, add a line to the user's cron -# table to call the (re)launch script every CRON_RELAUNCH_INTVL_MNTS mi- -# nutes. -# -#----------------------------------------------------------------------- -# -if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then -# -# Make a backup copy of the user's crontab file and save it in a file. -# - time_stamp=$( $DATE_UTIL "+%F_%T" ) - crontab_backup_fp="$EXPTDIR/crontab.bak.${time_stamp}" - print_info_msg "$VERBOSE" " -Copying contents of user cron table to backup file: - crontab_backup_fp = \"${crontab_backup_fp}\"" - - called_from_cron=${called_from_cron:-"FALSE"} - get_crontab_contents called_from_cron=${called_from_cron} \ - outvarname_crontab_cmd="crontab_cmd" \ - outvarname_crontab_contents="crontab_contents" - # To create the backup crontab file and add a new job to the user's - # existing cron table, use the "printf" command, not "echo", to print - # out variables. This is because "echo" will add a newline at the end - # of its output even if its input argument is a null string, resulting - # in extranous blank lines in the backup crontab file and/or the cron - # table itself. Using "printf" prevents the appearance of these blank - # lines. - printf "%s" "${crontab_contents}" > "${crontab_backup_fp}" - # - # Below, we use "grep" to determine whether the crontab line that the - # variable CRONTAB_LINE contains is already present in the cron table. - # For that purpose, we need to escape the asterisks in the string in - # CRONTAB_LINE with backslashes. Do this next. - # - crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ - $SED -r -e "s%[*]%\\\\*%g" ) - # - # In the grep command below, the "^" at the beginning of the string - # passed to grep is a start-of-line anchor, and the "$" at the end is - # an end-of-line anchor. Thus, in order for grep to find a match on - # any given line of the cron table's contents, that line must contain - # exactly the string in the variable crontab_line_esc_astr without any - # leading or trailing characters. This is to eliminate situations in - # which a line in the cron table contains the string in crontab_line_esc_astr - # but is precedeeded, for example, by the comment character "#" (in which - # case cron ignores that line) and/or is followed by further commands - # that are not part of the string in crontab_line_esc_astr (in which - # case it does something more than the command portion of the string in - # crontab_line_esc_astr does). - # - if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - grep_output=$( grep "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" ) - else - grep_output=$( printf "%s" "${crontab_contents}" | grep "^${crontab_line_esc_astr}$" ) - fi - exit_status=$? - - if [ "${exit_status}" -eq 0 ]; then - - print_info_msg " -The following line already exists in the cron table and thus will not be -added: - CRONTAB_LINE = \"${CRONTAB_LINE}\"" - - else - - print_info_msg "$VERBOSE" " -Adding the following line to the user's cron table in order to automatically -resubmit SRW workflow: - CRONTAB_LINE = \"${CRONTAB_LINE}\"" - - if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - printf "%s\n" "${CRONTAB_LINE}" >> "/u/$USER/cron/mycrontab" - else - # Add a newline to the end of crontab_contents only if it is not empty. - # This is needed so that when CRONTAB_LINE is printed out, it appears on - # a separate line. - crontab_contents=${crontab_contents:+"${crontab_contents}"$'\n'} - # When printing CRONTAB_LINE, add a newline at the end. This is necessary - # on certain machines (e.g. Cheyenne) while on others, it doesn't make - # a difference. - ( printf "%s" "${crontab_contents}"; printf "%s\n" "${CRONTAB_LINE}" ) | ${crontab_cmd} - fi - - fi - -fi -# -#----------------------------------------------------------------------- -# -# Create the FIXam directory under the experiment directory. In NCO mode, -# this will be a symlink to the directory specified in FIXgsm, while in -# community mode, it will be an actual directory with files copied into -# it from FIXgsm. -# -#----------------------------------------------------------------------- -# -# First, consider NCO mode. -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - ln_vrfy -fsn "$FIXgsm" "$FIXam" -# -# Resolve the target directory that the FIXam symlink points to and check -# that it exists. -# - path_resolved=$( $READLINK -m "$FIXam" ) - if [ ! -d "${path_resolved}" ]; then - print_err_msg_exit "\ -In order to be able to generate a forecast experiment in NCO mode (i.e. -when RUN_ENVIR set to \"nco\"), the path specified by FIXam after resolving -all symlinks (path_resolved) must be an existing directory (but in this -case isn't): - RUN_ENVIR = \"${RUN_ENVIR}\" - FIXam = \"$FIXam\" - path_resolved = \"${path_resolved}\" -Please ensure that path_resolved is an existing directory and then rerun -the experiment generation script." - fi -# -# Now consider community mode. -# -else - - print_info_msg "$VERBOSE" " -Copying fixed files from system directory (FIXgsm) to a subdirectory -(FIXam) in the experiment directory: - FIXgsm = \"$FIXgsm\" - FIXam = \"$FIXam\"" - - check_for_preexist_dir_file "$FIXam" "delete" - mkdir_vrfy -p "$FIXam" - mkdir_vrfy -p "$FIXam/fix_co2_proj" - - num_files=${#FIXgsm_FILES_TO_COPY_TO_FIXam[@]} - for (( i=0; i<${num_files}; i++ )); do - fn="${FIXgsm_FILES_TO_COPY_TO_FIXam[$i]}" - cp_vrfy "$FIXgsm/$fn" "$FIXam/$fn" - done - -fi -# -#----------------------------------------------------------------------- -# -# Copy MERRA2 aerosol climatology data. -# -#----------------------------------------------------------------------- -# -if [ "${USE_MERRA_CLIMO}" = "TRUE" ]; then - print_info_msg "$VERBOSE" " -Copying MERRA2 aerosol climatology data files from system directory -(FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: - FIXaer = \"${FIXaer}\" - FIXlut = \"${FIXlut}\" - FIXclim = \"${FIXclim}\"" - - check_for_preexist_dir_file "${FIXclim}" "delete" - mkdir_vrfy -p "${FIXclim}" - - cp_vrfy "${FIXaer}/merra2.aerclim"*".nc" "${FIXclim}/" - cp_vrfy "${FIXlut}/optics"*".dat" "${FIXclim}/" -fi -# -#----------------------------------------------------------------------- -# -# Copy templates of various input files to the experiment directory. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Copying templates of various input files to the experiment directory..." - -print_info_msg "$VERBOSE" " - Copying the template data table file to the experiment directory..." -cp_vrfy "${DATA_TABLE_TMPL_FP}" "${DATA_TABLE_FP}" - -print_info_msg "$VERBOSE" " - Copying the template field table file to the experiment directory..." -cp_vrfy "${FIELD_TABLE_TMPL_FP}" "${FIELD_TABLE_FP}" - -print_info_msg "$VERBOSE" " - Copying the template NEMS configuration file to the experiment directory..." -cp_vrfy "${NEMS_CONFIG_TMPL_FP}" "${NEMS_CONFIG_FP}" -# -# Copy the CCPP physics suite definition file from its location in the -# clone of the FV3 code repository to the experiment directory (EXPT- -# DIR). -# -print_info_msg "$VERBOSE" " -Copying the CCPP physics suite definition XML file from its location in -the forecast model directory sturcture to the experiment directory..." -cp_vrfy "${CCPP_PHYS_SUITE_IN_CCPP_FP}" "${CCPP_PHYS_SUITE_FP}" -# -# Copy the field dictionary file from its location in the -# clone of the FV3 code repository to the experiment directory (EXPT- -# DIR). -# -print_info_msg "$VERBOSE" " -Copying the field dictionary file from its location in the forecast -model directory sturcture to the experiment directory..." -cp_vrfy "${FIELD_DICT_IN_UWM_FP}" "${FIELD_DICT_FP}" -# -#----------------------------------------------------------------------- -# -# Set parameters in the FV3-LAM namelist file. -# -#----------------------------------------------------------------------- -# -print_info_msg " -Setting parameters in weather model's namelist file (FV3_NML_FP): - FV3_NML_FP = \"${FV3_NML_FP}\"" -# -# Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. -# These need to be set in the FV3-LAM Fortran namelist file. They represent -# the number of cell vertices in the x and y directions on the regional -# grid. -# -npx=$((NX+1)) -npy=$((NY+1)) -# -# For the physics suites that use RUC LSM, set the parameter kice to 9, -# Otherwise, leave it unspecified (which means it gets set to the default -# value in the forecast model). -# -# NOTE: -# May want to remove kice from FV3.input.yml (and maybe input.nml.FV3). -# -kice="" -if [ "${SDF_USES_RUC_LSM}" = "TRUE" ]; then - kice="9" -fi -# -# Set lsoil, which is the number of input soil levels provided in the -# chgres_cube output NetCDF file. This is the same as the parameter -# nsoill_out in the namelist file for chgres_cube. [On the other hand, -# the parameter lsoil_lsm (not set here but set in input.nml.FV3 and/or -# FV3.input.yml) is the number of soil levels that the LSM scheme in the -# forecast model will run with.] Here, we use the same approach to set -# lsoil as the one used to set nsoill_out in exregional_make_ics.sh. -# See that script for details. -# -# NOTE: -# May want to remove lsoil from FV3.input.yml (and maybe input.nml.FV3). -# Also, may want to set lsm here as well depending on SDF_USES_RUC_LSM. -# -lsoil="4" -if [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" -o \ - "${EXTRN_MDL_NAME_ICS}" = "RAP" ] && \ - [ "${SDF_USES_RUC_LSM}" = "TRUE" ]; then - lsoil="9" -fi -# -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the namelist variables that are physics- -# suite-independent need to be set to. Below, this variable will be -# passed to a python script that will in turn set the values of these -# variables in the namelist file. -# -# IMPORTANT: -# If we want a namelist variable to be removed from the namelist file, -# in the "settings" variable below, we need to set its value to the -# string "null". This is equivalent to setting its value to -# !!python/none -# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the -# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. -# -# It turns out that setting the variable to an empty string also works -# to remove it from the namelist! Which is better to use?? -# -settings="\ -'atmos_model_nml': { - 'blocksize': $BLOCKSIZE, - 'ccpp_suite': ${CCPP_PHYS_SUITE}, - } -'fv_core_nml': { - 'target_lon': ${LON_CTR}, - 'target_lat': ${LAT_CTR}, - 'nrows_blend': ${HALO_BLEND}, -# -# Question: -# For a ESGgrid type grid, what should stretch_fac be set to? This depends -# on how the FV3 code uses the stretch_fac parameter in the namelist file. -# Recall that for a ESGgrid, it gets set in the function set_gridparams_ESGgrid(.sh) -# to something like 0.9999, but is it ok to set it to that here in the -# FV3 namelist file? -# - 'stretch_fac': ${STRETCH_FAC}, - 'npx': $npx, - 'npy': $npy, - 'layout': [${LAYOUT_X}, ${LAYOUT_Y}], - 'bc_update_interval': ${LBC_SPEC_INTVL_HRS}, - } -'gfs_physics_nml': { - 'kice': ${kice:-null}, - 'lsoil': ${lsoil:-null}, - 'do_shum': ${DO_SHUM}, - 'do_sppt': ${DO_SPPT}, - 'do_skeb': ${DO_SKEB}, - 'do_spp': ${DO_SPP}, - 'n_var_spp': ${N_VAR_SPP}, - 'n_var_lndp': ${N_VAR_LNDP}, - 'lndp_type': ${LNDP_TYPE}, - 'fhcyc': ${FHCYC_LSM_SPP_OR_NOT}, - }" -# -# Add to "settings" the values of those namelist variables that specify -# the paths to fixed files in the FIXam directory. As above, these namelist -# variables are physcs-suite-independent. -# -# Note that the array FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING contains -# the mapping between the namelist variables and the names of the files -# in the FIXam directory. Here, we loop through this array and process -# each element to construct each line of "settings". -# -settings="$settings -'namsfc': {" - -dummy_run_dir="$EXPTDIR/any_cyc" -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - dummy_run_dir="${dummy_run_dir}/any_ensmem" -fi - -regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" -num_nml_vars=${#FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[@]} -for (( i=0; i<${num_nml_vars}; i++ )); do - - mapping="${FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[$i]}" - nml_var_name=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\1/p" ) - FIXam_fn=$( printf "%s\n" "$mapping" | - $SED -n -r -e "s/${regex_search}/\2/p" ) - - fp="\"\"" - if [ ! -z "${FIXam_fn}" ]; then - fp="$FIXam/${FIXam_fn}" -# -# If not in NCO mode, for portability and brevity, change fp so that it -# is a relative path (relative to any cycle directory immediately under -# the experiment directory). -# - if [ "${RUN_ENVIR}" != "nco" ]; then - fp=$( realpath --canonicalize-missing --relative-to="${dummy_run_dir}" "$fp" ) - fi - fi -# -# Add a line to the variable "settings" that specifies (in a yaml-compliant -# format) the name of the current namelist variable and the value it should -# be set to. -# - settings="$settings - '${nml_var_name}': $fp," - -done -# -# Add the closing curly bracket to "settings". -# -settings="$settings - }" -# -# Use netCDF4 when running the North American 3-km domain due to file size. -# -if [ "${PREDEF_GRID_NAME}" = "RRFS_NA_3km" ]; then -settings="$settings -'fms2_io_nml': { - 'netcdf_default_format': netcdf4, - }" -fi -# -# Add the relevant tendency-based stochastic physics namelist variables to -# "settings" when running with SPPT, SHUM, or SKEB turned on. If running -# with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only -# include an empty "nam_stochy" stanza. -# -settings="$settings -'nam_stochy': {" -if [ "${DO_SPPT}" = "TRUE" ]; then - settings="$settings - 'iseed_sppt': ${ISEED_SPPT}, - 'new_lscale': ${NEW_LSCALE}, - 'sppt': ${SPPT_MAG}, - 'sppt_logit': ${SPPT_LOGIT}, - 'sppt_lscale': ${SPPT_LSCALE}, - 'sppt_sfclimit': ${SPPT_SFCLIMIT}, - 'sppt_tau': ${SPPT_TSCALE}, - 'spptint': ${SPPT_INT}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," -fi - -if [ "${DO_SHUM}" = "TRUE" ]; then - settings="$settings - 'iseed_shum': ${ISEED_SHUM}, - 'new_lscale': ${NEW_LSCALE}, - 'shum': ${SHUM_MAG}, - 'shum_lscale': ${SHUM_LSCALE}, - 'shum_tau': ${SHUM_TSCALE}, - 'shumint': ${SHUM_INT}," -fi - -if [ "${DO_SKEB}" = "TRUE" ]; then - settings="$settings - 'iseed_skeb': ${ISEED_SKEB}, - 'new_lscale': ${NEW_LSCALE}, - 'skeb': ${SKEB_MAG}, - 'skeb_lscale': ${SKEB_LSCALE}, - 'skebnorm': ${SKEBNORM}, - 'skeb_tau': ${SKEB_TSCALE}, - 'skebint': ${SKEB_INT}, - 'skeb_vdof': ${SKEB_VDOF}," -fi - -if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then - settings="$settings - 'new_lscale': ${NEW_LSCALE}," -fi -settings="$settings - }" -# -# Add the relevant SPP namelist variables to "settings" when running with -# SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. -# -settings="$settings -'nam_sppperts': {" -if [ "${DO_SPP}" = "TRUE" ]; then - settings="$settings - 'iseed_spp': [ $( printf "%s, " "${ISEED_SPP[@]}" ) ], - 'spp_lscale': [ $( printf "%s, " "${SPP_LSCALE[@]}" ) ], - 'spp_prt_list': [ $( printf "%s, " "${SPP_MAG_LIST[@]}" ) ], - 'spp_sigtop1': [ $( printf "%s, " "${SPP_SIGTOP1[@]}" ) ], - 'spp_sigtop2': [ $( printf "%s, " "${SPP_SIGTOP2[@]}" ) ], - 'spp_stddev_cutoff': [ $( printf "%s, " "${SPP_STDDEV_CUTOFF[@]}" ) ], - 'spp_tau': [ $( printf "%s, " "${SPP_TSCALE[@]}" ) ], - 'spp_var_list': [ $( printf "%s, " "${SPP_VAR_LIST[@]}" ) ]," -fi -settings="$settings - }" -# -# Add the relevant LSM SPP namelist variables to "settings" when running with -# LSM SPP turned on. -# -settings="$settings -'nam_sfcperts': {" -if [ "${DO_LSM_SPP}" = "TRUE" ]; then - settings="$settings - 'lndp_type': ${LNDP_TYPE}, - 'lndp_model_type': ${LNDP_MODEL_TYPE}, - 'lndp_tau': [ $( printf "%s, " "${LSM_SPP_TSCALE[@]}" ) ], - 'lndp_lscale': [ $( printf "%s, " "${LSM_SPP_LSCALE[@]}" ) ], - 'iseed_lndp': [ $( printf "%s, " "${ISEED_LSM_SPP[@]}" ) ], - 'lndp_var_list': [ $( printf "%s, " "${LSM_SPP_VAR_LIST[@]}" ) ], - 'lndp_prt_list': [ $( printf "%s, " "${LSM_SPP_MAG_LIST[@]}" ) ]," -fi -settings="$settings - }" -print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the weather model's -namelist variables has been set as follows: - -settings = -$settings" -# -#----------------------------------------------------------------------- -# -# Call the set_namelist.py script to create a new FV3 namelist file (full -# path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as -# the base (i.e. starting) namelist file, with physics-suite-dependent -# modifications to the base file specified in the yaml configuration file -# FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), -# and with additional physics-suite-independent modificaitons specified -# in the variable "settings" set above. -# -#----------------------------------------------------------------------- -# -$USHDIR/set_namelist.py -q \ - -n ${FV3_NML_BASE_SUITE_FP} \ - -c ${FV3_NML_YAML_CONFIG_FP} ${CCPP_PHYS_SUITE} \ - -u "$settings" \ - -o ${FV3_NML_FP} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to generate an FV3 namelist file -failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = \"${FV3_NML_BASE_SUITE_FP}\" - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = \"${FV3_NML_YAML_CONFIG_FP}\" - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" - Full path to output namelist file: - FV3_NML_FP = \"${FV3_NML_FP}\" - Namelist settings specified on command line: - settings = -$settings" -# -# If not running the MAKE_GRID_TN task (which implies the workflow will -# use pregenerated grid files), set the namelist variables specifying -# the paths to surface climatology files. These files are located in -# (or have symlinks that point to them) in the FIXLAM directory. -# -# Note that if running the MAKE_GRID_TN task, this action usually cannot -# be performed here but must be performed in that task because the names -# of the surface climatology files depend on the CRES parameter (which is -# the C-resolution of the grid), and this parameter is in most workflow -# configurations is not known until the grid is created. -# -if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - - set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ -Call to function to set surface climatology file names in the FV3 namelist -file failed." - -fi -# -#----------------------------------------------------------------------- -# -# To have a record of how this experiment/workflow was generated, copy -# the experiment/workflow configuration file to the experiment directo- -# ry. -# -#----------------------------------------------------------------------- -# -cp_vrfy $USHDIR/${EXPT_CONFIG_FN} $EXPTDIR -# -#----------------------------------------------------------------------- -# -# For convenience, print out the commands that need to be issued on the -# command line in order to launch the workflow and to check its status. -# Also, print out the line that should be placed in the user's cron table -# in order for the workflow to be continually resubmitted. -# -#----------------------------------------------------------------------- -# -if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then - wflow_db_fn="${WFLOW_XML_FN%.xml}.db" - rocotorun_cmd="rocotorun -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" - rocotostat_cmd="rocotostat -w ${WFLOW_XML_FN} -d ${wflow_db_fn} -v 10" -fi - -print_info_msg " -======================================================================== -======================================================================== - -Experiment generation completed. The experiment directory is: - - EXPTDIR=\"$EXPTDIR\" - -======================================================================== -======================================================================== -" -# -#----------------------------------------------------------------------- -# -# If rocoto is required, print instructions on how to load and use it -# -#----------------------------------------------------------------------- -# -if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then - - print_info_msg "\ -To launch the workflow, first ensure that you have a compatible version -of rocoto available. For most pre-configured platforms, rocoto can be -loaded via a module: - - > module load rocoto - -For more details on rocoto, see the User's Guide. - -To launch the workflow, first ensure that you have a compatible version -of rocoto loaded. For example, to load version 1.3.1 of rocoto, use - - > module load rocoto/1.3.1 - -(This version has been tested on hera; later versions may also work but -have not been tested.) - -To launch the workflow, change location to the experiment directory -(EXPTDIR) and issue the rocotrun command, as follows: - - > cd $EXPTDIR - > ${rocotorun_cmd} - -To check on the status of the workflow, issue the rocotostat command -(also from the experiment directory): - - > ${rocotostat_cmd} - -Note that: - -1) The rocotorun command must be issued after the completion of each - task in the workflow in order for the workflow to submit the next - task(s) to the queue. - -2) In order for the output of the rocotostat command to be up-to-date, - the rocotorun command must be issued immediately before issuing the - rocotostat command. - -For automatic resubmission of the workflow (say every 3 minutes), the -following line can be added to the user's crontab (use \"crontab -e\" to -edit the cron table): - -*/3 * * * * cd $EXPTDIR && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" -" - -fi -# -# If necessary, run the NOMADS script to source external model data. -# -if [ "${NOMADS}" = "TRUE" ]; then - echo "Getting NOMADS online data" - echo "NOMADS_file_type=" $NOMADS_file_type - cd $EXPTDIR - $USHDIR/NOMADS_get_extrn_mdl_files.sh $DATE_FIRST_CYCL $CYCL_HRS $NOMADS_file_type $FCST_LEN_HRS $LBC_SPEC_INTVL_HRS -fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - -} - -# -#----------------------------------------------------------------------- -# -# Start of the script that will call the experiment/workflow generation -# function defined above. -# -#----------------------------------------------------------------------- -# -set -u -#set -x -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -if [[ $(uname -s) == Darwin ]]; then - command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } - scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) -else - scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -fi -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# -ushdir="${scrfunc_dir}" -# -# Set the name of and full path to the temporary file in which we will -# save some experiment/workflow variables. The need for this temporary -# file is explained below. -# -tmp_fn="tmp" -tmp_fp="$ushdir/${tmp_fn}" -rm -f "${tmp_fp}" -# -# Set the name of and full path to the log file in which the output from -# the experiment/workflow generation function will be saved. -# -log_fn="log.generate_FV3LAM_wflow" -log_fp="$ushdir/${log_fn}" -rm -f "${log_fp}" -# -# Call the generate_FV3LAM_wflow function defined above to generate the -# experiment/workflow. Note that we pipe the output of the function -# (and possibly other commands) to the "tee" command in order to be able -# to both save it to a file and print it out to the screen (stdout). -# The piping causes the call to the function (and the other commands -# grouped with it using the curly braces, { ... }) to be executed in a -# subshell. As a result, the experiment/workflow variables that the -# function sets are not available outside of the grouping, i.e. they are -# not available at and after the call to "tee". Since some of these va- -# riables are needed after the call to "tee" below, we save them in a -# temporary file and read them in outside the subshell later below. -# -{ -generate_FV3LAM_wflow 2>&1 # If this exits with an error, the whole {...} group quits, so things don't work... -retval=$? -echo "$EXPTDIR" >> "${tmp_fp}" -echo "$retval" >> "${tmp_fp}" -} | tee "${log_fp}" -# -# Read in experiment/workflow variables needed later below from the tem- -# porary file created in the subshell above containing the call to the -# generate_FV3LAM_wflow function. These variables are not directly -# available here because the call to generate_FV3LAM_wflow above takes -# place in a subshell (due to the fact that we are then piping its out- -# put to the "tee" command). Then remove the temporary file. -# -exptdir=$( sed "1q;d" "${tmp_fp}" ) -retval=$( sed "2q;d" "${tmp_fp}" ) -rm "${tmp_fp}" -# -# If the call to the generate_FV3LAM_wflow function above was success- -# ful, move the log file in which the "tee" command saved the output of -# the function to the experiment directory. -# -if [[ $retval == 0 ]]; then - mv "${log_fp}" "$exptdir" -# -# If the call to the generate_FV3LAM_wflow function above was not suc- -# cessful, print out an error message and exit with a nonzero return -# code. -# -else - printf " -Experiment generation failed. Check the log file from the experiment -generation script in the file specified by log_fp: - log_fp = \"${log_fp}\" -Stopping. -" - exit 1 -fi diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index cbb434c69e..3f885ad978 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 import os +import sys import unittest +import argparse from datetime import datetime from python_utils import import_vars, set_env_var, print_input_args, \ - run_command, define_macos_utilities, check_var_valid_value -from constants import valid_vals_BOOLEAN + run_command, define_macos_utilities, print_info_msg def get_crontab_contents(called_from_cron): """ @@ -36,15 +37,10 @@ def get_crontab_contents(called_from_cron): """ print_input_args(locals()) - - #import all env vars + + #import selected env vars IMPORTS = ["MACHINE", "USER"] import_vars(env_vars=IMPORTS) - - # - # Make sure called_from_cron is set to a valid value. - # - check_var_valid_value(called_from_cron, valid_vals_BOOLEAN) if MACHINE == "WCOSS_DELL_P3": __crontab_cmd__="" @@ -62,17 +58,115 @@ def get_crontab_contents(called_from_cron): if called_from_cron: __crontab_cmd__="/usr/bin/crontab" (_,__crontab_contents__,_)=run_command(f'''{__crontab_cmd__} -l''') + + return __crontab_cmd__, __crontab_contents__ + +def add_crontab_line(): + """ Add crontab line to cron table """ + + #import selected env vars + IMPORTS = ["MACHINE", "USER", "CRONTAB_LINE", "VERBOSE", "EXPTDIR"] + import_vars(env_vars=IMPORTS) + # - # On Cheyenne, the output of the "crontab -l" command contains a 3-line - # header (comments) at the top that is not actually part of the user's - # cron table. This needs to be removed to avoid adding an unnecessary - # copy of this header to the user's cron table. + # Make a backup copy of the user's crontab file and save it in a file. # - if MACHINE == "CHEYENNE": - (_,__crontab_contents__,_)=run_command(f'''printf "%s" "{__crontab_contents__}" | tail -n +4 ''') + time_stamp = datetime.now().strftime("%F_%T") + crontab_backup_fp=os.path.join(EXPTDIR,f"crontab.bak.{time_stamp}") + print_info_msg(f''' + Copying contents of user cron table to backup file: + crontab_backup_fp = \"{crontab_backup_fp}\"''',verbose=VERBOSE) + + global called_from_cron + try: called_from_cron + except: called_from_cron = False + + # Get crontab contents + crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=called_from_cron) + + # Create backup + run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''') + + # Add crontab line + if CRONTAB_LINE in crontab_contents: + + print_info_msg(f''' + The following line already exists in the cron table and thus will not be + added: + CRONTAB_LINE = \"{CRONTAB_LINE}\"''') + + else: + + print_info_msg(f''' + Adding the following line to the user's cron table in order to automatically + resubmit SRW workflow: + CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE) + + #add new line to crontab contents if it doesn't have one + NEWLINE_CHAR="" + if crontab_contents and crontab_contents[-1] != "\n": + NEWLINE_CHAR="\n" + + #add the crontab line + if MACHINE == "WCOSS_DELL_P3": + run_command(f'''printf "%b%s\n" '{NEWLINE_CHAR}' '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''') + else: + run_command(f'''printf "%s%b%s\n" '{crontab_contents}' '{NEWLINE_CHAR}' '{CRONTAB_LINE}' | {crontab_cmd}''') + +def delete_crontab_line(called_from_cron): + """ Delete crontab line after job is complete i.e. either SUCCESS/FAILURE + but not IN PROGRESS status""" + + print_input_args(locals()) - return __crontab_cmd__, __crontab_contents__ + #import selected env vars + IMPORTS = ["MACHINE", "USER", "CRONTAB_LINE"] + import_vars(env_vars=IMPORTS) + + # + # Get the full contents of the user's cron table. + # + (crontab_cmd,crontab_contents) = get_crontab_contents(called_from_cron) + # + # Remove the line in the contents of the cron table corresponding to the + # current forecast experiment (if that line is part of the contents). + # Then record the results back into the user's cron table. + # + if (CRONTAB_LINE + '\n') in crontab_contents: + crontab_contents = crontab_contents.replace(CRONTAB_LINE+'\n','') + else: + crontab_contents = crontab_contents.replace(CRONTAB_LINE,'') + + if MACHINE == "WCOSS_DELL_P3": + run_command(f'''echo '{crontab_contents}' > "/u/{USER}/cron/mycrontab"''') + else: + run_command(f'''echo '{crontab_contents}' | {crontab_cmd}''') + +def parse_args(argv): + """ Parse command line arguments for deleting crontab line. + This is needed because it is called from shell script + """ + parser = argparse.ArgumentParser( + description='Crontab job manupilation program.' + ) + + parser.add_argument('-d', '--delete', + dest='delete', + action='store_true', + help='Delete crontab line.') + + parser.add_argument('-c', '--called-from-cron', + dest='called_from_cron', + action='store_true', + help='Called from cron.') + + return parser.parse_args(argv) +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + if args.delete: + delete_crontab_line(args.called_from_cron) + class Testing(unittest.TestCase): def test_get_crontab_contents(self): crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=True) diff --git a/ush/get_crontab_contents.sh b/ush/get_crontab_contents.sh deleted file mode 100644 index a7a00854ef..0000000000 --- a/ush/get_crontab_contents.sh +++ /dev/null @@ -1,74 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that returns the contents of the user's -# cron table as well as the command to use to manipulate the cron table -# (i.e. the "crontab" command, but on some platforms the version or -# location of this may change depending on other circumstances, e.g. on -# Cheyenne, this depends on whether a script that wants to call "crontab" -# is itself being called from a cron job). Arguments are as follows: -# -# called_from_cron: -# Boolean flag that specifies whether this function (and the scripts or -# functions that are calling it) are called as part of a cron job. Must -# be set to "TRUE" or "FALSE". -# -# outvarname_crontab_cmd: -# Name of the output variable that will contain the command to issue for -# the system "crontab" command. -# -# outvarname_crontab_contents: -# Name of the output variable that will contain the contents of the -# user's cron table. -# -#----------------------------------------------------------------------- -# -function get_crontab_contents() { - - { save_shell_opts; set -u +x; } > /dev/null 2>&1 - - local valid_args=( \ - "called_from_cron" \ - "outvarname_crontab_cmd" \ - "outvarname_crontab_contents" \ - ) - process_args valid_args "$@" - print_input_args "valid_args" - - local __crontab_cmd__ \ - __crontab_contents__ - # - # Make sure called_from_cron is set to a valid value. - # - source $USHDIR/constants.sh - check_var_valid_value "called_from_cron" "valid_vals_BOOLEAN" - called_from_cron=$(boolify "${called_from_cron}") - - if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - __crontab_cmd__="" - __crontab_contents__=$( cat "/u/$USER/cron/mycrontab" ) - else - __crontab_cmd__="crontab" - # - # On Cheyenne, simply typing "crontab" will launch the crontab command - # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized - # version of crontab that will work if called from scripts that are - # themselves being called as cron jobs. In that case, we must instead - # call the system version of crontab at /usr/bin/crontab. - # - if [ "$MACHINE" = "CHEYENNE" ]; then - if [ -n "${called_from_cron}" ] && [ "${called_from_cron}" = "TRUE" ]; then - __crontab_cmd__="/usr/bin/crontab" - fi - fi - __crontab_contents__=$( ${__crontab_cmd__} -l ) - fi - # - # Set output variables. - # - printf -v ${outvarname_crontab_cmd} "%s" "${__crontab_cmd__}" - printf -v ${outvarname_crontab_contents} "%s" "${__crontab_contents__}" - - { restore_shell_opts; } > /dev/null 2>&1 - -} diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 8a4dbe6031..17734b9477 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -99,7 +99,6 @@ fi . $USHDIR/constants.sh . $USHDIR/source_util_funcs.sh . $USHDIR/init_env.sh -. $USHDIR/get_crontab_contents.sh # #----------------------------------------------------------------------- # @@ -393,35 +392,14 @@ script for this experiment: CRONTAB_LINE = \"${CRONTAB_LINE}\" " # -# Below, we use "grep" to determine whether the crontab line that the -# variable CRONTAB_LINE contains is already present in the cron table. -# For that purpose, we need to escape the asterisks in the string in -# CRONTAB_LINE with backslashes. Do this next. +# Remove CRONTAB_LINE from cron table # - crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \ - $SED -r -e "s%[*]%\\\\*%g" ) -# -# Get the full contents of the user's cron table. -# - get_crontab_contents called_from_cron=${called_from_cron} \ - outvarname_crontab_cmd="crontab_cmd" \ - outvarname_crontab_contents="crontab_contents" -# -# Remove the line in the contents of the cron table corresponding to the -# current forecast experiment (if that line is part of the contents). -# Then record the results back into the user's cron table. -# -# In the string passed to the grep command below, we use the line start -# and line end anchors ("^" and "$", respectively) to ensure that we -# only find lines in the crontab that contain exactly the string in -# crontab_line_esc_astr without any leading or trailing characters. -# - crontab_contents=$( echo "${crontab_contents}" | grep -v "^${crontab_line_esc_astr}$" ) - - if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - echo "${crontab_contents}" > "/u/$USER/cron/mycrontab" + if [ "${called_from_cron}" = "TRUE" ]; then + MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \ + python3 $USHDIR/get_crontab_contents.py --delete --called-from-cron else - echo "${crontab_contents}" | ${crontab_cmd} + MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \ + python3 $USHDIR/get_crontab_contents.py --delete fi fi diff --git a/ush/link_fix.py b/ush/link_fix.py old mode 100644 new mode 100755 index 9788a4ad47..56fccc8567 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -2,12 +2,14 @@ import unittest import os +import sys +import argparse import glob from python_utils import import_vars, set_env_var, print_input_args, \ print_info_msg, print_err_msg_exit, create_symlink_to_file, \ define_macos_utilities, check_var_valid_value, \ - cd_vrfy, mkdir_vrfy, find_pattern_in_str + cd_vrfy, mkdir_vrfy, find_pattern_in_str, load_shell_config def link_fix(verbose, file_group): """ This file defines a function that ... @@ -300,28 +302,10 @@ def link_fix(verbose, file_group): #----------------------------------------------------------------------- # if file_group == "grid": - target=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH4}.nc" symlink=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.nc" create_symlink_to_file(target,symlink,True) # - # The surface climatology file generation code looks for a grid file - # having a name of the form "C${GFDLgrid_RES}_grid.tile7.halo4.nc" (i.e. - # the C-resolution used in the name of this file is the number of grid - # points per horizontal direction per tile, just like in the global model). - # Thus, if we are running the MAKE_SFC_CLIMO_TN task, if the grid is of - # GFDLgrid type, and if we are not using GFDLgrid_RES in filenames (i.e. - # we are using the equivalent global uniform grid resolution instead), - # then create a link whose name uses the GFDLgrid_RES that points to the - # link whose name uses the equivalent global uniform resolution. - # - if RUN_TASK_MAKE_SFC_CLIMO and \ - GRID_GEN_METHOD == "GFDLgrid" and \ - not GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: - target=f"{cres}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.halo{NH4}.nc" - symlink=f"C{GFDLgrid_RES}{DOT_OR_USCORE}grid.tile{TILE_RGNL}.nc" - create_symlink_to_file(target,symlink,relative) - # #----------------------------------------------------------------------- # # If considering surface climatology files, create symlinks to the surface @@ -366,6 +350,30 @@ def link_fix(verbose, file_group): return res +def parse_args(argv): + """ Parse command line arguments""" + parser = argparse.ArgumentParser( + description='Creates symbolic links to FIX directories.' + ) + + parser.add_argument('-f', '--file-group', + dest='file_group', + required=True, + help='File group, could be one of ["grid", "orog", "sfc_climo"].') + + parser.add_argument('-p', '--path-to-defns', + dest='path_to_defns', + required=True, + help='Path to var_defns file.') + + return parser.parse_args(argv) + +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + cfg = load_shell_config(args.path_to_defns) + import_vars(dictionary=cfg) + link_fix(VERBOSE, args.file_group) + class Testing(unittest.TestCase): def test_link_fix(self): res = link_fix(verbose=True, file_group="grid") diff --git a/ush/link_fix.sh b/ush/link_fix.sh deleted file mode 100755 index 48bf0ca3f8..0000000000 --- a/ush/link_fix.sh +++ /dev/null @@ -1,460 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This file defines a function that ... -# -#----------------------------------------------------------------------- -# -function link_fix() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names that this script/function can -# accept. Then process the arguments provided to it (which should con- -# sist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ -"verbose" \ -"file_group" \ -"output_varname_res_in_filenames" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local valid_vals_verbose \ - valid_vals_file_group \ - fns \ - fps \ - run_task \ - sfc_climo_fields \ - num_fields \ - i \ - ii \ - res_prev \ - res \ - fp_prev \ - fp \ - fn \ - relative_link_flag \ - cres \ - tmp \ - fns_sfc_climo_with_halo_in_fn \ - fns_sfc_climo_no_halo_in_fn \ - target \ - symlink -# -#----------------------------------------------------------------------- -# -# Set the valid values that various input arguments can take on and then -# ensure that the values passed in are one of these valid values. -# -#----------------------------------------------------------------------- -# - valid_vals_verbose=( "TRUE" "FALSE" ) - check_var_valid_value "verbose" "valid_vals_verbose" - - valid_vals_file_group=( "grid" "orog" "sfc_climo" ) - check_var_valid_value "file_group" "valid_vals_file_group" -# -#----------------------------------------------------------------------- -# -# Create symlinks in the FIXLAM directory pointing to the grid files. -# These symlinks are needed by the make_orog, make_sfc_climo, make_ic, -# make_lbc, and/or run_fcst tasks. -# -# Note that we check that each target file exists before attempting to -# create symlinks. This is because the "ln" command will create sym- -# links to non-existent targets without returning with a nonzero exit -# code. -# -#----------------------------------------------------------------------- -# - print_info_msg "$verbose" " -Creating links in the FIXLAM directory to the grid files..." -# -#----------------------------------------------------------------------- -# -# Create globbing patterns for grid, orography, and surface climatology -# files. -# -# -# For grid files (i.e. file_group set to "grid"), symlinks are created -# in the FIXLAM directory to files (of the same names) in the GRID_DIR. -# These symlinks/files and the reason each is needed is listed below: -# -# 1) "C*.mosaic.halo${NHW}.nc" -# This mosaic file for the wide-halo grid (i.e. the grid with a ${NHW}- -# cell-wide halo) is needed as an input to the orography filtering -# executable in the orography generation task. The filtering code -# extracts from this mosaic file the name of the file containing the -# grid on which it will generate filtered topography. Note that the -# orography generation and filtering are both performed on the wide- -# halo grid. The filtered orography file on the wide-halo grid is then -# shaved down to obtain the filtered orography files with ${NH3}- and -# ${NH4}-cell-wide halos. -# -# The raw orography generation step in the make_orog task requires the -# following symlinks/files: -# -# a) C*.mosaic.halo${NHW}.nc -# The script for the make_orog task extracts the name of the grid -# file from this mosaic file; this name should be -# "C*.grid.tile${TILE_RGNL}.halo${NHW}.nc". -# -# b) C*.grid.tile${TILE_RGNL}.halo${NHW}.nc -# This is the -# The script for the make_orog task passes the name of the grid -# file (extracted above from the mosaic file) to the orography -# generation executable. The executable then -# reads in this grid file and generates a raw orography -# file on the grid. The raw orography file is initially renamed "out.oro.nc", -# but for clarity, it is then renamed "C*.raw_orog.tile${TILE_RGNL}.halo${NHW}.nc". -# -# c) The fixed files thirty.second.antarctic.new.bin, landcover30.fixed, -# and gmted2010.30sec.int. -# -# The orography filtering step in the make_orog task requires the -# following symlinks/files: -# -# a) C*.mosaic.halo${NHW}.nc -# This is the mosaic file for the wide-halo grid. The orography -# filtering executable extracts from this file the name of the grid -# file containing the wide-halo grid (which should be -# "${CRES}.grid.tile${TILE_RGNL}.halo${NHW}.nc"). The executable then -# looks for this grid file IN THE DIRECTORY IN WHICH IT IS RUNNING. -# Thus, before running the executable, the script creates a symlink in this run directory that -# points to the location of the actual wide-halo grid file. -# -# b) C*.raw_orog.tile${TILE_RGNL}.halo${NHW}.nc -# This is the raw orography file on the wide-halo grid. The script -# for the make_orog task copies this file to a new file named -# "C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc" that will be -# used as input to the orography filtering executable. The executable -# will then overwrite the contents of this file with the filtered orography. -# Thus, the output of the orography filtering executable will be -# the file C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc. -# -# The shaving step in the make_orog task requires the following: -# -# a) C*.filtered_orog.tile${TILE_RGNL}.halo${NHW}.nc -# This is the filtered orography file on the wide-halo grid. -# This gets shaved down to two different files: -# -# i) ${CRES}.oro_data.tile${TILE_RGNL}.halo${NH0}.nc -# This is the filtered orography file on the halo-0 grid. -# -# ii) ${CRES}.oro_data.tile${TILE_RGNL}.halo${NH4}.nc -# This is the filtered orography file on the halo-4 grid. -# -# Note that the file names of the shaved files differ from that of -# the initial unshaved file on the wide-halo grid in that the field -# after ${CRES} is now "oro_data" (not "filtered_orog") to comply -# with the naming convention used more generally. -# -# 2) "C*.mosaic.halo${NH4}.nc" -# This mosaic file for the grid with a 4-cell-wide halo is needed as -# an input to the surface climatology generation executable. The -# surface climatology generation code reads from this file the number -# of tiles (which should be 1 for a regional grid) and the tile names. -# More importantly, using the ESMF function ESMF_GridCreateMosaic(), -# it creates a data object of type esmf_grid; the grid information -# in this object is obtained from the grid file specified in the mosaic -# file, which should be "C*.grid.tile${TILE_RGNL}.halo${NH4}.nc". The -# dimensions specified in this grid file must match the ones specified -# in the (filtered) orography file "C*.oro_data.tile${TILE_RGNL}.halo${NH4}.nc" -# that is also an input to the surface climatology generation executable. -# If they do not, then the executable will crash with an ESMF library -# error (something like "Arguments are incompatible"). -# -# Thus, for the make_sfc_climo task, the following symlinks/files must -# exist: -# a) "C*.mosaic.halo${NH4}.nc" -# b) "C*.grid.tile${TILE_RGNL}.halo${NH4}.nc" -# c) "C*.oro_data.tile${TILE_RGNL}.halo${NH4}.nc" -# -# 3) -# -# -#----------------------------------------------------------------------- -# - case "${file_group}" in -# - "grid") - fns=( \ - "C*${DOT_OR_USCORE}mosaic.halo${NHW}.nc" \ - "C*${DOT_OR_USCORE}mosaic.halo${NH4}.nc" \ - "C*${DOT_OR_USCORE}mosaic.halo${NH3}.nc" \ - "C*${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NHW}.nc" \ - "C*${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" \ - "C*${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" \ - ) - fps=( "${fns[@]/#/${GRID_DIR}/}" ) - run_task="${RUN_TASK_MAKE_GRID}" - ;; -# - "orog") - fns=( \ - "C*${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc" \ - "C*${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" \ - ) - if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then - fns+=( \ - "C*${DOT_OR_USCORE}oro_data_ss.tile${TILE_RGNL}.halo${NH0}.nc" \ - "C*${DOT_OR_USCORE}oro_data_ls.tile${TILE_RGNL}.halo${NH0}.nc" \ - ) - fi - fps=( "${fns[@]/#/${OROG_DIR}/}" ) - run_task="${RUN_TASK_MAKE_OROG}" - ;; -# -# The following list of symlinks (which have the same names as their -# target files) need to be created made in order for the make_ics and -# make_lbcs tasks (i.e. tasks involving chgres_cube) to work. -# - "sfc_climo") - num_fields=${#SFC_CLIMO_FIELDS[@]} - fns=() - for (( i=0; i<${num_fields}; i++ )); do - ii=$((2*i)) - fns[$ii]="C*.${SFC_CLIMO_FIELDS[$i]}.tile${TILE_RGNL}.halo${NH0}.nc" - fns[$ii+1]="C*.${SFC_CLIMO_FIELDS[$i]}.tile${TILE_RGNL}.halo${NH4}.nc" - done - fps=( "${fns[@]/#/${SFC_CLIMO_DIR}/}" ) - run_task="${RUN_TASK_MAKE_SFC_CLIMO}" - ;; -# - esac -# -#----------------------------------------------------------------------- -# -# Find all files matching the globbing patterns and make sure that they -# all have the same resolution (an integer) in their names. -# -#----------------------------------------------------------------------- -# - i=0 - res_prev="" - res="" - fp_prev="" - - for fp in ${fps[@]}; do - - fn=$( basename $fp ) - - res=$( printf "%s" $fn | $SED -n -r -e "s/^C([0-9]*).*/\1/p" ) - if [ -z $res ]; then - print_err_msg_exit "\ -The resolution could not be extracted from the current file's name. The -full path to the file (fp) is: - fp = \"${fp}\" -This may be because fp contains the * globbing character, which would -imply that no files were found that match the globbing pattern specified -in fp." - fi - - if [ $i -gt 0 ] && [ ${res} != ${res_prev} ]; then - print_err_msg_exit "\ -The resolutions (as obtained from the file names) of the previous and -current file (fp_prev and fp, respectively) are different: - fp_prev = \"${fp_prev}\" - fp = \"${fp}\" -Please ensure that all files have the same resolution." - fi - - i=$((i+1)) - fp_prev="$fp" - res_prev=${res} - - done -# -#----------------------------------------------------------------------- -# -# If the output variable name is not set to a null string, set it. This -# variable is just the resolution extracted from the file names in the -# specified file group. Note that if the output variable name is not -# specified in the call to this function, the process_args function will -# set it to a null string, in which case no output variable will be set. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${output_varname_res_in_filenames}" ]; then - eval ${output_varname_res_in_filenames}="$res" - fi -# -#----------------------------------------------------------------------- -# -# Replace the * globbing character in the set of globbing patterns with -# the resolution. This will result in a set of (full paths to) specific -# files. -# -#----------------------------------------------------------------------- -# - fps=( "${fps[@]/\*/$res}" ) -# -#----------------------------------------------------------------------- -# -# In creating the various symlinks below, it is convenient to work in -# the FIXLAM directory. We will change directory back to the original -# later below. -# -#----------------------------------------------------------------------- -# - cd_vrfy "$FIXLAM" -# -#----------------------------------------------------------------------- -# -# Use the set of full file paths generated above as the link targets to -# create symlinks to these files in the FIXLAM directory. -# -#----------------------------------------------------------------------- -# -# If the task in consideration (which will be one of the pre-processing -# tasks MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN) was run, then -# the target files will be located under the experiment directory. In -# this case, we use relative symlinks in order the experiment directory -# more portable and the symlinks more readable. However, if the task -# was not run, then pregenerated grid, orography, or surface climatology -# files will be used, and those will be located in an arbitrary directory -# (specified by the user) that is somwehere outside the experiment -# directory. Thus, in this case, there isn't really an advantage to using -# relative symlinks, so we use symlinks with absolute paths. -# - if [ "${run_task}" = "TRUE" ]; then - relative_link_flag="TRUE" - else - relative_link_flag="FALSE" - fi - - for fp in "${fps[@]}"; do - fn=$( basename $fp ) - create_symlink_to_file target="$fp" symlink="$fn" \ - relative="${relative_link_flag}" - done -# -#----------------------------------------------------------------------- -# -# Set the C-resolution based on the resolution appearing in the file -# names. -# -#----------------------------------------------------------------------- -# - cres="C$res" -# -#----------------------------------------------------------------------- -# -# If considering grid files, create a symlink to the halo4 grid file -# that does not contain the halo size in its name. This is needed by -# the tasks that generate the initial and lateral boundary condition -# files. -# -#----------------------------------------------------------------------- -# - if [ "${file_group}" = "grid" ]; then - target="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" - symlink="${cres}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.nc" - create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" - fi -# -#----------------------------------------------------------------------- -# -# If considering surface climatology files, create symlinks to the surface -# climatology files that do not contain the halo size in their names. -# These are needed by the task that generates the initial condition files. -# -#----------------------------------------------------------------------- -# - if [ "${file_group}" = "sfc_climo" ]; then - - tmp=( "${SFC_CLIMO_FIELDS[@]/#/${cres}.}" ) - fns_sfc_climo_with_halo_in_fn=( "${tmp[@]/%/.tile${TILE_RGNL}.halo${NH4}.nc}" ) - fns_sfc_climo_no_halo_in_fn=( "${tmp[@]/%/.tile${TILE_RGNL}.nc}" ) - - for (( i=0; i<${num_fields}; i++ )); do - target="${fns_sfc_climo_with_halo_in_fn[$i]}" - symlink="${fns_sfc_climo_no_halo_in_fn[$i]}" - create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" - done -# -# In order to be able to specify the surface climatology file names in -# the forecast model's namelist file, in the FIXLAM directory a symlink -# must be created for each surface climatology field that has "tile1" in -# its name (and no "halo") and which points to the corresponding "tile7.halo0" -# file. -# - tmp=( "${SFC_CLIMO_FIELDS[@]/#/${cres}.}" ) - fns_sfc_climo_tile7_halo0_in_fn=( "${tmp[@]/%/.tile${TILE_RGNL}.halo${NH0}.nc}" ) - fns_sfc_climo_tile1_no_halo_in_fn=( "${tmp[@]/%/.tile1.nc}" ) - - for (( i=0; i<${num_fields}; i++ )); do - target="${fns_sfc_climo_tile7_halo0_in_fn[$i]}" - symlink="${fns_sfc_climo_tile1_no_halo_in_fn[$i]}" - create_symlink_to_file target="$target" symlink="$symlink" relative="TRUE" - done - - fi -# -#----------------------------------------------------------------------- -# -# Change directory back to original one. -# -#----------------------------------------------------------------------- -# - cd_vrfy - -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the start of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index 330a9a6472..5b3849fdd0 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -187,41 +187,6 @@ # #----------------------------------------------------------------------- # -# The RRFS CONUS domain with ~13km cells. -# -#----------------------------------------------------------------------- -# -"RRFS_CONUS_13km": - GRID_GEN_METHOD: "ESGgrid" - ESGgrid_LON_CTR: -97.5 - ESGgrid_LAT_CTR: 38.5 - ESGgrid_DELX: 13000.0 - ESGgrid_DELY: 13000.0 - ESGgrid_NX: 396 - ESGgrid_NY: 232 - ESGgrid_PAZI: 0.0 - ESGgrid_WIDE_HALO_WIDTH: 6 - DT_ATMOS: 45 - LAYOUT_X: 16 - LAYOUT_Y: 10 - BLOCKSIZE: 32 - #if QUILTING = True - WRTCMP_write_groups: 1 - WRTCMP_write_tasks_per_group: 10 - WRTCMP_output_grid: "lambert_conformal" - WRTCMP_cen_lon: -97.5 - WRTCMP_cen_lat: 38.5 - WRTCMP_stdlat1: 38.5 - WRTCMP_stdlat2: 38.5 - WRTCMP_nx: 393 - WRTCMP_ny: 225 - WRTCMP_lon_lwr_left: -121.70231097 - WRTCMP_lat_lwr_left: 22.57417972 - WRTCMP_dx: 13000.0 - WRTCMP_dy: 13000.0 -# -#----------------------------------------------------------------------- -# # The RRFS CONUS domain with ~3km cells. # #----------------------------------------------------------------------- @@ -496,7 +461,7 @@ GFDLgrid_LON_T6_CTR: -97.5 GFDLgrid_LAT_T6_CTR: 38.5 GFDLgrid_STRETCH_FAC: 1.4 - GFDLgrid_RES: 96 + GFDLgrid_NUM_CELLS: 96 GFDLgrid_REFINE_RATIO: 3 num_margin_cells_T6_left: 12 GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: 13 @@ -506,7 +471,7 @@ GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: 17 num_margin_cells_T6_top: 16 GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: 80 - GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: True + GFDLgrid_USE_NUM_CELLS_IN_FILENAMES: True DT_ATMOS: 225 LAYOUT_X: 6 LAYOUT_Y: 4 @@ -539,7 +504,7 @@ GFDLgrid_LON_T6_CTR: -97.5 GFDLgrid_LAT_T6_CTR: 38.5 GFDLgrid_STRETCH_FAC: 1.5 - GFDLgrid_RES: 768 + GFDLgrid_NUM_CELLS: 768 GFDLgrid_REFINE_RATIO: 3 num_margin_cells_T6_left: 69 GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G: 70 @@ -549,7 +514,7 @@ GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G: 165 num_margin_cells_T6_top: 164 GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G: 604 - GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES: True + GFDLgrid_USE_NUM_CELLS_IN_FILENAMES: True DT_ATMOS: 18 LAYOUT_X: 30 LAYOUT_Y: 22 diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index 9371488d5c..3aa90e1549 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -1,22 +1,25 @@ -from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file -from .check_for_preexist_dir_file import check_for_preexist_dir_file -from .check_var_valid_value import check_var_valid_value -from .count_files import count_files -from .create_symlink_to_file import create_symlink_to_file -from .define_macos_utilities import define_macos_utilities -from .environment import str_to_date, date_to_str, str_to_type, type_to_str, list_to_str, \ - str_to_list, set_env_var, get_env_var, import_vars, export_vars -from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy -from .get_charvar_from_netcdf import get_charvar_from_netcdf -from .get_elem_inds import get_elem_inds -from .interpol_to_arbit_CRES import interpol_to_arbit_CRES -from .print_input_args import print_input_args -from .print_msg import print_info_msg, print_err_msg_exit -from .process_args import process_args -from .run_command import run_command -from .config_parser import load_yaml_config, cfg_to_yaml_str, \ - load_json_config, cfg_to_json_str, \ - load_ini_config, cfg_to_ini_str, get_ini_value, \ - load_shell_config, cfg_to_shell_str, \ - load_config_file -from .xml_parser import load_xml_file, has_tag_with_value +try: + from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file, flatten_dict + from .check_for_preexist_dir_file import check_for_preexist_dir_file + from .check_var_valid_value import check_var_valid_value + from .count_files import count_files + from .create_symlink_to_file import create_symlink_to_file + from .define_macos_utilities import define_macos_utilities + from .environment import str_to_date, date_to_str, str_to_type, type_to_str, list_to_str, \ + str_to_list, set_env_var, get_env_var, import_vars, export_vars + from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy + from .get_elem_inds import get_elem_inds + from .interpol_to_arbit_CRES import interpol_to_arbit_CRES + from .print_input_args import print_input_args + from .print_msg import print_info_msg, print_err_msg_exit + from .process_args import process_args + from .run_command import run_command + from .get_charvar_from_netcdf import get_charvar_from_netcdf + from .xml_parser import load_xml_file, has_tag_with_value + from .config_parser import load_shell_config, cfg_to_shell_str, \ + load_json_config, cfg_to_json_str, \ + load_ini_config, cfg_to_ini_str, \ + get_ini_value, load_config_file, \ + load_yaml_config, cfg_to_yaml_str +except: + pass diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index da7131df26..b343551bfd 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -14,7 +14,10 @@ """ import argparse -import yaml +try: + import yaml +except: + pass import json import sys import os @@ -49,7 +52,10 @@ def join_str(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) -yaml.add_constructor('!join_str', join_str, Loader=yaml.SafeLoader) +try: + yaml.add_constructor('!join_str', join_str, Loader=yaml.SafeLoader) +except: + pass ########## # JSON @@ -86,12 +92,15 @@ def load_shell_config(config_file): # Save env vars before and after sourcing the scipt and then # do a diff to get variables specifically defined/updated in the script # Method sounds brittle but seems to work ok so far + pid = os.getpid() code = dedent(f''' #!/bin/bash - (set -o posix; set) > ./_t1 + t1="./t1.{pid}" + t2="./t2.{pid}" + (set -o posix; set) > $t1 {{ . {config_file}; set +x; }} &>/dev/null - (set -o posix; set) > ./_t2 - diff ./_t1 ./_t2 | grep "> " | cut -c 3- - rm -rf ./_t1 ./_t2 + (set -o posix; set) > $t2 + diff $t1 $t2 | grep "> " | cut -c 3- + rm -rf $t1 $t2 ''') (_,config_str,_) = run_command(code) lines = config_str.splitlines() @@ -131,7 +140,7 @@ def load_ini_config(config_file): if not os.path.exists(config_file): print_err_msg_exit(f''' The specified configuration file does not exist: - \"{file_name}\"''') + \"{config_file}\"''') config = configparser.ConfigParser() config.read(config_file) diff --git a/ush/python_utils/environment.py b/ush/python_utils/environment.py index 25f03b8fd8..8123130557 100644 --- a/ush/python_utils/environment.py +++ b/ush/python_utils/environment.py @@ -4,25 +4,30 @@ import inspect import shlex from datetime import datetime, date +from types import ModuleType def str_to_date(s): """ Get python datetime object from string. - It tests for only two formats used in RRFS: YYYYMMDD and YYYYMMDDHHMM Args: s: a string Returns: datetime object or None """ + v = None try: - v = datetime.strptime(s, "%Y%m%d%H%M") - return v - except: - try: + l = len(s) + if l == 8: v = datetime.strptime(s, "%Y%m%d") - return v - except: - return None + if l == 10: + v = datetime.strptime(s, "%Y%m%d%H") + elif l == 12: + v = datetime.strptime(s, "%Y%m%d%H%M") + elif l == 14: + v = datetime.strptime(s, "%Y%m%d%H%M%S") + except: + v = None + return v def date_to_str(d,short=False): """ Get string from python datetime object. @@ -240,7 +245,9 @@ def export_vars(dictionary=None, source_dict=None, env_vars=None): # skip functions and other unlikely variable names if callable(v): continue - if not k or k.islower() or k[0] == '_': + if isinstance(v,ModuleType): + continue + if not k or k[0] == '_': continue dictionary[k] = list_to_str(v) diff --git a/ush/python_utils/misc.py b/ush/python_utils/misc.py index 1934ac3d6c..c299e02735 100644 --- a/ush/python_utils/misc.py +++ b/ush/python_utils/misc.py @@ -55,3 +55,21 @@ def find_pattern_in_file(pattern, file_name): return match.groups() return None +def flatten_dict(dictionary,keys=None): + """ Faltten a recursive dictionary (e.g.yaml/json) to be one level deep + Args: + dictionary: the source dictionary + keys: list of keys on top level whose contents to flatten, if None all of them + Returns: + A one-level deep dictionary for the selected set of keys + """ + flat_dict = {} + for k,v in dictionary.items(): + if not keys or k in keys: + if isinstance(v,dict): + r = flatten_dict(v) + flat_dict.update(r) + else: + flat_dict[k] = v + return flat_dict + diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 9d9ae4b39e..b56979374f 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -import unittest import os +import sys +import argparse +import unittest from textwrap import dedent from datetime import datetime from python_utils import print_input_args, print_info_msg, print_err_msg_exit,\ - date_to_str, mkdir_vrfy,cp_vrfy,\ - import_vars,set_env_var,\ - define_macos_utilities, cfg_to_yaml_str + date_to_str, mkdir_vrfy, cp_vrfy, str_to_type, \ + import_vars,set_env_var, \ + define_macos_utilities, cfg_to_yaml_str, \ + load_shell_config from set_namelist import set_namelist @@ -109,6 +112,25 @@ def set_FV3nml_ens_stoch_seeds(cdate): settings = {settings_str}''')) +def parse_args(argv): + """ Parse command line arguments""" + parser = argparse.ArgumentParser( + description='Creates stochastic seeds for an ensemble experiment.' + ) + + parser.add_argument('-c', '--cdate', + dest='cdate', + required=True, + help='Date.') + + return parser.parse_args(argv) + +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + cfg = load_shell_config(args.path_to_defns) + import_vars(dictionary=cfg) + set_FV3nml_ens_stoch_seeds(str_to_type(args.cdate)) + class Testing(unittest.TestCase): def test_set_FV3nml_ens_stoch_seeds(self): set_FV3nml_ens_stoch_seeds(cdate=self.cdate) @@ -116,22 +138,23 @@ def setUp(self): define_macos_utilities(); set_env_var('DEBUG',True) set_env_var('VERBOSE',True) + self.cdate=datetime(2021, 1, 1) USHDIR = os.path.dirname(os.path.abspath(__file__)) EXPTDIR = os.path.join(USHDIR,"test_data","expt"); cp_vrfy(os.path.join(USHDIR,f'templates{os.sep}input.nml.FV3'), \ os.path.join(EXPTDIR,'input.nml')) - self.cdate=datetime(2021, 1, 1) + for i in range(2): + mkdir_vrfy("-p", os.path.join(EXPTDIR,f"{date_to_str(self.cdate,True)}{os.sep}mem{i+1}")) - mkdir_vrfy("-p", os.path.join(EXPTDIR,f'{date_to_str(self.cdate,True)}{os.sep}mem0')) set_env_var("USHDIR",USHDIR) set_env_var("CYCLE_BASEDIR",EXPTDIR) - set_env_var("ENSMEM_INDX",0) + set_env_var("ENSMEM_INDX",2) set_env_var("FV3_NML_FN","input.nml") set_env_var("FV3_NML_FP",os.path.join(EXPTDIR,"input.nml")) set_env_var("DO_SPP",True) set_env_var("DO_SHUM",True) set_env_var("DO_SKEB",True) set_env_var("DO_LSM_SPP",True) - ISEED_SPP = [ 4, 4, 4, 4, 4] + ISEED_SPP = [ 4, 5, 6, 7, 8] set_env_var("ISEED_SPP",ISEED_SPP) diff --git a/ush/set_FV3nml_ens_stoch_seeds.sh b/ush/set_FV3nml_ens_stoch_seeds.sh deleted file mode 100644 index 64322d6648..0000000000 --- a/ush/set_FV3nml_ens_stoch_seeds.sh +++ /dev/null @@ -1,187 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that, for an ensemble-enabled experiment -# (i.e. for an experiment for which the workflow configuration variable -# DO_ENSEMBLE has been set to "TRUE"), creates new namelist files with -# unique stochastic "seed" parameters, using a base namelist file in the -# ${EXPTDIR} directory as a template. These new namelist files are stored -# within each member directory housed within each cycle directory. Files -# of any two ensemble members differ only in their stochastic "seed" -# parameter values. These namelist files are generated when this file is -# called as part of the RUN_FCST_TN task. -# -#----------------------------------------------------------------------- -# -function set_FV3nml_ens_stoch_seeds() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "cdate" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local i \ - ensmem_num \ - fv3_nml_ens_fp \ - iseed_shum \ - iseed_skeb \ - iseed_sppt \ - iseed_spp \ - iseed_lsm_spp \ - num_iseed_spp \ - num_iseed_lsm_spp \ - settings -# -#----------------------------------------------------------------------- -# -# For a given cycle and member, generate a namelist file with unique -# seed values. -# -#----------------------------------------------------------------------- -# - ensmem_name="mem${ENSMEM_INDX}" - - fv3_nml_ensmem_fp="${CYCLE_BASEDIR}/${cdate}/${ensmem_name}/${FV3_NML_FN}" - - ensmem_num=$((10#${ENSMEM_INDX})) - - settings="\ -'nam_stochy': {" - - if [ ${DO_SPPT} = TRUE ]; then - - iseed_sppt=$(( cdate*1000 + ensmem_num*10 + 1 )) - settings="$settings - 'iseed_sppt': ${iseed_sppt}," - - fi - - if [ ${DO_SHUM} = TRUE ]; then - - iseed_shum=$(( cdate*1000 + ensmem_num*10 + 2 )) - settings="$settings - 'iseed_shum': ${iseed_shum}," - - fi - - if [ ${DO_SKEB} = TRUE ]; then - - iseed_skeb=$(( cdate*1000 + ensmem_num*10 + 3 )) - settings="$settings - 'iseed_skeb': ${iseed_skeb}," - - fi - settings="$settings - }" - - settings="$settings -'nam_sppperts': {" - - if [ ${DO_SPP} = TRUE ]; then - - num_iseed_spp=${#ISEED_SPP[@]} - for (( i=0; i<${num_iseed_spp}; i++ )); do - iseed_spp[$i]=$(( cdate*1000 + ensmem_num*10 + ${ISEED_SPP[$i]} )) - done - - settings="$settings - 'iseed_spp': [ $( printf "%s, " "${iseed_spp[@]}" ) ]," - - fi - - settings="$settings - }" - - settings="$settings -'nam_sfcperts': {" - - if [ ${DO_LSM_SPP} = TRUE ]; then - - iseed_lsm_spp=$(( cdate*1000 + ensmem_num*10 + 9)) - - settings="$settings - 'iseed_lndp': [ $( printf "%s, " "${iseed_lsm_spp[@]}" ) ]," - - fi - - settings="$settings - }" - - $USHDIR/set_namelist.py -q \ - -n ${FV3_NML_FP} \ - -u "$settings" \ - -o ${fv3_nml_ensmem_fp} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the FV3 -namelist file that specify the paths to the surface climatology files -failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_FP = \"${FV3_NML_FP}\" - Full path to output namelist file: - fv3_nml_ensmem_fp = \"${fv3_nml_ensmem_fp}\" - Namelist settings specified on command line (these have highest precedence): - settings = -$settings" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 518f81ae36..c1a4d68c65 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -2,11 +2,13 @@ import unittest import os +import sys +import argparse from textwrap import dedent from python_utils import print_input_args, print_info_msg, print_err_msg_exit,\ check_var_valid_value,mv_vrfy,mkdir_vrfy,cp_vrfy,\ - rm_vrfy,import_vars,set_env_var,\ + rm_vrfy,import_vars,set_env_var,load_shell_config,\ define_macos_utilities,find_pattern_in_str,cfg_to_yaml_str from set_namelist import set_namelist @@ -92,6 +94,25 @@ def set_FV3nml_sfc_climo_filenames(): rm_vrfy(f'{fv3_nml_base_fp}') +def parse_args(argv): + """ Parse command line arguments""" + parser = argparse.ArgumentParser( + description='Set surface climatology fields.' + ) + + parser.add_argument('-p', '--path-to-defns', + dest='path_to_defns', + required=True, + help='Path to var_defns file.') + + return parser.parse_args(argv) + +if __name__ == '__main__': + args = parse_args(sys.argv[1:]) + cfg = load_shell_config(args.path_to_defns) + import_vars(dictionary=cfg) + set_FV3nml_sfc_climo_filenames() + class Testing(unittest.TestCase): def test_set_FV3nml_sfc_climo_filenames(self): set_FV3nml_sfc_climo_filenames() diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh deleted file mode 100644 index d6f9dc5d4c..0000000000 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ /dev/null @@ -1,231 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that sets the values of the variables in -# the forecast model's namelist file that specify the paths to the surface -# climatology files on the FV3LAM native grid (which are either pregenerated -# or created by the MAKE_SFC_CLIMO_TN task). Note that the workflow -# generation scripts create symlinks to these surface climatology files -# in the FIXLAM directory, and the values in the namelist file that get -# set by this function are relative or full paths to these links. -# -#----------------------------------------------------------------------- -# -function set_FV3nml_sfc_climo_filenames() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=() - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local regex_search \ - suffix \ - num_nml_vars \ - mapping \ - nml_var_name \ - sfc_climo_field_name \ - fp -# -#----------------------------------------------------------------------- -# -# In the forecast model's namelist file, set those variables representing -# the name of a fixed file that has associated with it a surface climatology -# file to the path to that surface climatology file. -# -# Note: -# The following symlinks that contain no "halo" in their names currently -# point to the halo4 surface climatology files. But it is not clear whether -# these should be pointing to the halo0 or halo4 files. Ask!!! -# -#----------------------------------------------------------------------- -# -# The regular expression regex_search set below will be used to extract -# from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING -# the name of the namelist variable to set and the corresponding surface -# climatology field from which to form the name of the surface climatology -# file. This regular expression matches any string that consists of the -# following sequence: -# -# 1) Zero or more spaces at the beginning of the string, followed by -# 2) A sequence of one or more characters that does not include a space -# or a pipe (i.e. the "|" character; this sequence is the namelist -# variable name), followed by -# 3) Zero or more spaces, followed by -# 4) A pipe, followed by -# 5) A sequence of one or more characters that does not include a space -# or a pipe (this sequence is the surface climatology field associated -# with the namelist variable), followed by -# 6) Zero or more spaces at the end of the string. -# -regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" -# -# Set the suffix of the surface climatology files. -# -# Questions: -# 1) Should we be using the halo0 or halo4 files? -# 2) For clarity, is it possible to use the actual name of the file (i.e. -# the actual ending that is either "tile7.halo0.nc" or "tile7.halo4.nc" -# instead of "tileX.nc"? -# -#suffix="tile${TILE_RGNL}.halo4.nc" -suffix="tileX.nc" -# -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the namelist variables that specify the -# surface climatology file paths need to be set to (one namelist variable -# per line, plus a header and footer). Below, this variable will be -# passed to a python script that will create the namelist file. -# -# Note that the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING contains -# the mapping between the namelist variables and the surface climatology -# fields. Here, we loop through this array and process each element to -# construct each line of "settings". -# -settings="\ -'namsfc': {" - -dummy_run_dir="$EXPTDIR/any_cyc" -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - dummy_run_dir="${dummy_run_dir}/any_ensmem" -fi - -num_nml_vars=${#FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING[@]} -for (( i=0; i<${num_nml_vars}; i++ )); do - - mapping="${FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING[$i]}" - nml_var_name=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\1/p" ) - sfc_climo_field_name=$( printf "%s\n" "$mapping" | - $SED -n -r -e "s/${regex_search}/\2/p" ) -# -# Check that the surface climatology field associated with the current -# namelist variable is valid. -# - check_var_valid_value "sfc_climo_field_name" "SFC_CLIMO_FIELDS" -# -# Set the full path to the surface climatology file. -# - fp="${FIXLAM}/${CRES}.${sfc_climo_field_name}.$suffix" -# -# If not in NCO mode, for portability and brevity change fp so that it -# is a relative path (relative to any cycle directory immediately under -# the experiment directory). -# - if [ "${RUN_ENVIR}" != "nco" ]; then - fp=$( realpath --canonicalize-missing --relative-to="${dummy_run_dir}" "$fp" ) - fi -# -# Add a line to the variable "settings" that specifies (in a yaml-compliant -# format) the name of the current namelist variable and the value it should -# be set to. -# - settings="$settings - '${nml_var_name}': $fp," - -done - -settings="$settings - }" -# -# For debugging purposes, print out what "settings" has been set to. -# -print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the namelist variables -has been set as follows: - -settings = -$settings" -# -#----------------------------------------------------------------------- -# -# Rename the FV3 namelist file for the experiment by appending the string -# ".base" to its name. The call to the set_namelist.py script below will -# use this file as the base (i.e. starting) namelist file, and it will -# modify it as specified by the varaible "settings" above, saving the -# result in a new FV3 namelist file for the experiment. Once this is -# done, we remove the base namelist file since it is no longer needed. -# -#----------------------------------------------------------------------- -# -fv3_nml_base_fp="${FV3_NML_FP}.base" -mv_vrfy "${FV3_NML_FP}" "${fv3_nml_base_fp}" - -$USHDIR/set_namelist.py -q \ - -n ${fv3_nml_base_fp} \ - -u "$settings" \ - -o ${FV3_NML_FP} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the FV3 -namelist file that specify the paths to the surface climatology files -failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_nml_base_fp = \"${fv3_nml_base_fp}\" - Full path to output namelist file: - FV3_NML_FP = \"${FV3_NML_FP}\" - Namelist settings specified on command line (these have highest precedence): - settings = -$settings" - -rm_vrfy "${fv3_nml_base_fp}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/set_cycle_dates.sh b/ush/set_cycle_dates.sh deleted file mode 100644 index e13c78d7ee..0000000000 --- a/ush/set_cycle_dates.sh +++ /dev/null @@ -1,145 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that, given the starting date (date_start, -# in the form YYYYMMDD), the ending date (date_end, in the form YYYYMMDD), -# and an array containing the cycle hours for each day (whose elements -# have the form HH), returns an array of cycle date-hours whose elements -# have the form YYYYMMDD. Here, YYYY is a four-digit year, MM is a two- -# digit month, DD is a two-digit day of the month, and HH is a two-digit -# hour of the day. -# -#----------------------------------------------------------------------- -# -function set_cycle_dates() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ -"date_start" \ -"date_end" \ -"cycle_hrs" \ -"incr_cycl_freq" \ -"output_varname_all_cdates" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local all_cdates date_crnt -# -#----------------------------------------------------------------------- -# -# Ensure that the ending date is at or after the starting date. -# -#----------------------------------------------------------------------- -# - if [ "${date_end}" -lt "${date_start}" ]; then - print_err_msg_exit "\ -End date (date_end) must be at or after start date (date_start): - date_start = \"${date_start}\" - date_end = \"${date_end}\"" - fi -# -#----------------------------------------------------------------------- -# -# In the following "while" loop, we begin with the starting date and -# increment by n days (incr_day) each time through the loop until we reach the ending -# date. For each date, we obtain an intermediate array of cdates (whose -# elements have the format YYYYMMDDHH) by prepending to the elements of -# cycle_hrs the current date. (Thus, this array has the same number of -# elements as cycle_hrs.) We then append this intermediate array to the -# final array that will contain all cdates (i.e. over all days and cycle -# hours). -# -#----------------------------------------------------------------------- -# - all_cdates=() - date_crnt="${date_start}" - - if [ "${incr_cycl_freq}" -le 24 ]; then - incr_day=1 - else - incr_day=$(( ${incr_cycl_freq} / 24 )) - incr_day_rem=$(( ${incr_cycl_freq} % 24 )) - - if [ "${incr_day_rem}" -gt 0 ];then - print_err_msg_exit "\ -INCR_CYCL_FREQ is not divided by 24: - INCR_CYCL_FREQ = \"${incr_cycl_freq}\"" - fi - fi - - while [ "${date_crnt}" -le "${date_end}" ]; do - all_cdates+=( $( printf "%s " ${cycle_hrs[@]/#/${date_crnt}} ) ) - date_crnt=$( $DATE_UTIL -d "${date_crnt} + ${incr_day} days" +%Y%m%d ) - done -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - all_cdates_str="( "$( printf "\"%s\" " "${all_cdates[@]}" )")" - eval ${output_varname_all_cdates}=${all_cdates_str} -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh deleted file mode 100644 index 12b11e88da..0000000000 --- a/ush/set_extrn_mdl_params.sh +++ /dev/null @@ -1,35 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file sets some parameters that are model or mode specific. -# -#----------------------------------------------------------------------- -# -function set_extrn_mdl_params() { - # - #----------------------------------------------------------------------- - # - # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift - # the starting time of the external model that provides lateral boundary - # conditions. - # - #----------------------------------------------------------------------- - # - case "${EXTRN_MDL_NAME_LBCS}" in - "RAP") - EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"3"} - ;; - *) - EXTRN_MDL_LBCS_OFFSET_HRS=${EXTRN_MDL_LBCS_OFFSET_HRS:-"0"} - ;; - esac -} - -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# -set_extrn_mdl_params diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py index c1ead9522f..722c2f27e2 100644 --- a/ush/set_gridparams_ESGgrid.py +++ b/ush/set_gridparams_ESGgrid.py @@ -3,7 +3,6 @@ import unittest from datetime import datetime,timedelta -from constants import radius_Earth,degs_per_radian from python_utils import import_vars, set_env_var, print_input_args def set_gridparams_ESGgrid(lon_ctr,lat_ctr,nx,ny,halo_width,delx,dely,pazi): @@ -24,6 +23,10 @@ def set_gridparams_ESGgrid(lon_ctr,lat_ctr,nx,ny,halo_width,delx,dely,pazi): """ print_input_args(locals()) + + # get needed environment variables + IMPORTS = ['RADIUS_EARTH', 'DEGS_PER_RADIAN'] + import_vars(env_vars=IMPORTS) # #----------------------------------------------------------------------- # @@ -57,8 +60,8 @@ def set_gridparams_ESGgrid(lon_ctr,lat_ctr,nx,ny,halo_width,delx,dely,pazi): # #----------------------------------------------------------------------- # - del_angle_x_sg = (delx / (2.0 * radius_Earth)) * degs_per_radian - del_angle_y_sg = (dely / (2.0 * radius_Earth)) * degs_per_radian + del_angle_x_sg = (delx / (2.0 * RADIUS_EARTH)) * DEGS_PER_RADIAN + del_angle_y_sg = (dely / (2.0 * RADIUS_EARTH)) * DEGS_PER_RADIAN neg_nx_of_dom_with_wide_halo = -(nx + 2 * halo_width) neg_ny_of_dom_with_wide_halo = -(ny + 2 * halo_width) # @@ -69,10 +72,10 @@ def set_gridparams_ESGgrid(lon_ctr,lat_ctr,nx,ny,halo_width,delx,dely,pazi): #----------------------------------------------------------------------- # return (lon_ctr,lat_ctr,nx,ny,pazi,halo_width,stretch_factor, - "{:0.10f}".format(del_angle_x_sg), - "{:0.10f}".format(del_angle_y_sg), - "{:.0f}".format(neg_nx_of_dom_with_wide_halo), - "{:.0f}".format(neg_ny_of_dom_with_wide_halo)) + del_angle_x_sg, + del_angle_y_sg, + int(neg_nx_of_dom_with_wide_halo), + int(neg_ny_of_dom_with_wide_halo)) class Testing(unittest.TestCase): def test_set_gridparams_ESGgrid(self): @@ -93,18 +96,18 @@ def test_set_gridparams_ESGgrid(self): self.assertEqual(\ (LON_CTR,LAT_CTR,NX,NY,PAZI,NHW,STRETCH_FAC, - DEL_ANGLE_X_SG, - DEL_ANGLE_Y_SG, + round(DEL_ANGLE_X_SG,10), + round(DEL_ANGLE_Y_SG,10), NEG_NX_OF_DOM_WITH_WIDE_HALO, NEG_NY_OF_DOM_WITH_WIDE_HALO), (-97.5, 38.5, 1748, 1038, 0.0, 6,0.999, - "0.0134894006", - "0.0134894006", - "-1760", - "-1050") + 0.0134894006, + 0.0134894006, + -1760, + -1050) ) def setUp(self): - set_env_var('DEBUG',True) - set_env_var('VERBOSE',True) + set_env_var('RADIUS_EARTH',6371200.0) + set_env_var('DEGS_PER_RADIAN',57.2957795131) diff --git a/ush/set_gridparams_ESGgrid.sh b/ush/set_gridparams_ESGgrid.sh deleted file mode 100644 index 4b81377713..0000000000 --- a/ush/set_gridparams_ESGgrid.sh +++ /dev/null @@ -1,215 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets the parameters -# for a grid that is to be generated using the "ESGgrid" grid generation -# method (i.e. GRID_GEN_METHOD set to "ESGgrid"). -# -#----------------------------------------------------------------------- -# -function set_gridparams_ESGgrid() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# - local homerrfs=${scrfunc_dir%/*} - local ushdir="$homerrfs/ush" -# -#----------------------------------------------------------------------- -# -# Source the file containing various mathematical, physical, etc constants. -# -#----------------------------------------------------------------------- -# - . $ushdir/constants.sh -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. -# Then process the arguments provided to this script/function (which -# should consist of a set of name-value pairs of the form arg1="value1", -# etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "lon_ctr" \ - "lat_ctr" \ - "nx" \ - "ny" \ - "halo_width" \ - "delx" \ - "dely" \ - "pazi" \ - "outvarname_lon_ctr" \ - "outvarname_lat_ctr" \ - "outvarname_nx" \ - "outvarname_ny" \ - "outvarname_pazi" \ - "outvarname_halo_width" \ - "outvarname_stretch_factor" \ - "outvarname_del_angle_x_sg" \ - "outvarname_del_angle_y_sg" \ - "outvarname_neg_nx_of_dom_with_wide_halo" \ - "outvarname_neg_ny_of_dom_with_wide_halo" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# -# print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local stretch_factor \ - del_angle_x_sg \ - del_angle_y_sg \ - neg_nx_of_dom_with_wide_halo \ - neg_ny_of_dom_with_wide_halo -# -#----------------------------------------------------------------------- -# -# For a ESGgrid-type grid, the orography filtering is performed by pass- -# ing to the orography filtering the parameters for an "equivalent" glo- -# bal uniform cubed-sphere grid. These are the parameters that a global -# uniform cubed-sphere grid needs to have in order to have a nominal -# grid cell size equal to that of the (average) cell size on the region- -# al grid. These globally-equivalent parameters include a resolution -# (in units of number of cells in each of the two horizontal directions) -# and a stretch factor. The equivalent resolution is calculated in the -# script that generates the grid, and the stretch factor needs to be set -# to 1 because we are considering an equivalent globally UNIFORM grid. -# However, it turns out that with a non-symmetric regional grid (one in -# which nx is not equal to ny), setting stretch_factor to 1 fails be- -# cause the orography filtering program is designed for a global cubed- -# sphere grid and thus assumes that nx and ny for a given tile are equal -# when stretch_factor is exactly equal to 1. <-- Why is this? Seems like symmetry btwn x and y should still hold when the stretch factor is not equal to 1. -# It turns out that the program will work if we set stretch_factor to a -# value that is not exactly 1. This is what we do below. -# -#----------------------------------------------------------------------- -# - stretch_factor="0.999" # Check whether the orography program has been fixed so that we can set this to 1... -# -#----------------------------------------------------------------------- -# -# Set parameters needed as inputs to the regional_grid grid generation -# code. -# -#----------------------------------------------------------------------- -# - del_angle_x_sg=$( bc -l <<< "(${delx}/(2.0*${radius_Earth}))*${degs_per_radian}" ) - del_angle_x_sg=$( printf "%0.10f\n" ${del_angle_x_sg} ) - - del_angle_y_sg=$( bc -l <<< "(${dely}/(2.0*${radius_Earth}))*${degs_per_radian}" ) - del_angle_y_sg=$( printf "%0.10f\n" ${del_angle_y_sg} ) - - neg_nx_of_dom_with_wide_halo=$( bc -l <<< "-($nx + 2*${halo_width})" ) - neg_nx_of_dom_with_wide_halo=$( printf "%.0f\n" ${neg_nx_of_dom_with_wide_halo} ) - - neg_ny_of_dom_with_wide_halo=$( bc -l <<< "-($ny + 2*${halo_width})" ) - neg_ny_of_dom_with_wide_halo=$( printf "%.0f\n" ${neg_ny_of_dom_with_wide_halo} ) -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_lon_ctr}" ]; then - printf -v ${outvarname_lon_ctr} "%s" "${lon_ctr}" - fi - - if [ ! -z "${outvarname_lat_ctr}" ]; then - printf -v ${outvarname_lat_ctr} "%s" "${lat_ctr}" - fi - - if [ ! -z "${outvarname_nx}" ]; then - printf -v ${outvarname_nx} "%s" "${nx}" - fi - - if [ ! -z "${outvarname_ny}" ]; then - printf -v ${outvarname_ny} "%s" "${ny}" - fi - - if [ ! -z "${outvarname_halo_width}" ]; then - printf -v ${outvarname_halo_width} "%s" "${halo_width}" - fi - - if [ ! -z "${outvarname_stretch_factor}" ]; then - printf -v ${outvarname_stretch_factor} "%s" "${stretch_factor}" - fi - - if [ ! -z "${outvarname_pazi}" ]; then - printf -v ${outvarname_pazi} "%s" "${pazi}" - fi - - if [ ! -z "${outvarname_del_angle_x_sg}" ]; then - printf -v ${outvarname_del_angle_x_sg} "%s" "${del_angle_x_sg}" - fi - - if [ ! -z "${outvarname_del_angle_y_sg}" ]; then - printf -v ${outvarname_del_angle_y_sg} "%s" "${del_angle_y_sg}" - fi - - if [ ! -z "${outvarname_neg_nx_of_dom_with_wide_halo}" ]; then - printf -v ${outvarname_neg_nx_of_dom_with_wide_halo} "%s" "${neg_nx_of_dom_with_wide_halo}" - fi - - if [ ! -z "${outvarname_neg_ny_of_dom_with_wide_halo}" ]; then - printf -v ${outvarname_neg_ny_of_dom_with_wide_halo} "%s" "${neg_ny_of_dom_with_wide_halo}" - fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py index 0dab11f756..40a74d8105 100644 --- a/ush/set_gridparams_GFDLgrid.py +++ b/ush/set_gridparams_GFDLgrid.py @@ -2,8 +2,6 @@ import unittest -from constants import radius_Earth,degs_per_radian - from python_utils import import_vars, set_env_var, print_input_args, \ print_info_msg, print_err_msg_exit diff --git a/ush/set_gridparams_GFDLgrid.sh b/ush/set_gridparams_GFDLgrid.sh deleted file mode 100644 index 68b6937be1..0000000000 --- a/ush/set_gridparams_GFDLgrid.sh +++ /dev/null @@ -1,571 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets the parameters -# for a grid that is to be generated using the "GFDLgrid" grid genera- -# tion method (i.e. GRID_GEN_METHOD set to "GFDLgrid"). -# -#----------------------------------------------------------------------- -# -function set_gridparams_GFDLgrid() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# - local homerrfs=${scrfunc_dir%/*} - local ushdir="$homerrfs/ush" -# -#----------------------------------------------------------------------- -# -# Source the file containing various mathematical, physical, etc constants. -# -#----------------------------------------------------------------------- -# - . $ushdir/constants.sh -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. -# Then process the arguments provided to this script/function (which -# should consist of a set of name-value pairs of the form arg1="value1", -# etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "lon_of_t6_ctr" \ - "lat_of_t6_ctr" \ - "res_of_t6g" \ - "stretch_factor" \ - "refine_ratio_t6g_to_t7g" \ - "istart_of_t7_on_t6g" \ - "iend_of_t7_on_t6g" \ - "jstart_of_t7_on_t6g" \ - "jend_of_t7_on_t6g" \ - "verbose" \ - "outvarname_lon_of_t7_ctr" \ - "outvarname_lat_of_t7_ctr" \ - "outvarname_nx_of_t7_on_t7g" \ - "outvarname_ny_of_t7_on_t7g" \ - "outvarname_halo_width_on_t7g" \ - "outvarname_stretch_factor" \ - "outvarname_istart_of_t7_with_halo_on_t6sg" \ - "outvarname_iend_of_t7_with_halo_on_t6sg" \ - "outvarname_jstart_of_t7_with_halo_on_t6sg" \ - "outvarname_jend_of_t7_with_halo_on_t6sg" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local nx_of_t6_on_t6g \ - ny_of_t6_on_t6g \ - num_left_margin_cells_on_t6g \ - num_right_margin_cells_on_t6g \ - num_bot_margin_cells_on_t6g \ - num_top_margin_cells_on_t6g \ - lon_of_t7_ctr \ - lat_of_t7_ctr \ - istart_of_t7_on_t6sg \ - iend_of_t7_on_t6sg \ - jstart_of_t7_on_t6sg \ - jend_of_t7_on_t6sg \ - halo_width_on_t7g \ - halo_width_on_t6sg \ - istart_of_t7_with_halo_on_t6sg \ - iend_of_t7_with_halo_on_t6sg \ - jstart_of_t7_with_halo_on_t6sg \ - jend_of_t7_with_halo_on_t6sg \ - halo_width_on_t6sg \ - halo_width_on_t6g \ - halo_width_on_t7g \ - nx_of_t7_on_t6sg \ - nx_of_t7_on_t6g \ - nx_of_t7_on_t7g \ - ny_of_t7_on_t6sg \ - ny_of_t7_on_t6g \ - ny_of_t7_on_t7g \ - nx_of_t6_on_t6sg \ - ny_of_t6_on_t6sg \ - prime_factors_nx_of_t7_on_t7g \ - prime_factors_ny_of_t7_on_t7g \ - nx_of_t7_with_halo_on_t6sg \ - nx_of_t7_with_halo_on_t6g \ - nx_of_t7_with_halo_on_t7g \ - ny_of_t7_with_halo_on_t6sg \ - ny_of_t7_with_halo_on_t6g \ - ny_of_t7_with_halo_on_t7g -# -#----------------------------------------------------------------------- -# -# To simplify the grid setup, we require that tile 7 be centered on tile -# 6. Note that this is not really a restriction because tile 6 can al- -# ways be moved so that it is centered on tile 7 [the location of tile 6 -# doesn't really matter because for a regional setup, the forecast model -# will only run on tile 7 (not on tiles 1-6)]. -# -# We now check that tile 7 is centered on tile 6 by checking (1) that -# the number of cells (on tile 6) between the left boundaries of these -# two tiles is equal to that between their right boundaries and (2) that -# the number of cells (on tile 6) between the bottom boundaries of these -# two tiles is equal to that between their top boundaries. If not, we -# print out an error message and exit. If so, we set the longitude and -# latitude of the center of tile 7 to those of tile 6 and continue. -# -#----------------------------------------------------------------------- -# - nx_of_t6_on_t6g=${res_of_t6g} - ny_of_t6_on_t6g=${res_of_t6g} - - num_left_margin_cells_on_t6g=$(( istart_of_t7_on_t6g - 1 )) - num_right_margin_cells_on_t6g=$(( nx_of_t6_on_t6g - iend_of_t7_on_t6g )) - -# This if-statement can hopefully be removed once EMC agrees to make their -# GFDLgrid type grids (tile 7) symmetric about tile 6. - if [ ${num_left_margin_cells_on_t6g} -ne ${num_right_margin_cells_on_t6g} ]; then - print_err_msg_exit "\ -In order for tile 7 to be centered in the x direction on tile 6, the x- -direction tile 6 cell indices at which tile 7 starts and ends (given by -istart_of_t7_on_t6g and iend_of_t7_on_t6g, respectively) must be set -such that the number of tile 6 cells in the margin between the left -boundaries of tiles 6 and 7 (given by num_left_margin_cells_on_t6g) is -equal to that in the margin between their right boundaries (given by -num_right_margin_cells_on_t6g): - istart_of_t7_on_t6g = ${istart_of_t7_on_t6g} - iend_of_t7_on_t6g = ${iend_of_t7_on_t6g} - num_left_margin_cells_on_t6g = ${num_left_margin_cells_on_t6g} - num_right_margin_cells_on_t6g = ${num_right_margin_cells_on_t6g} -Note that the total number of cells in the x-direction on tile 6 is gi- -ven by: - nx_of_t6_on_t6g = ${nx_of_t6_on_t6g} -Please reset istart_of_t7_on_t6g and iend_of_t7_on_t6g and rerun." - fi - - num_bot_margin_cells_on_t6g=$(( jstart_of_t7_on_t6g - 1 )) - num_top_margin_cells_on_t6g=$(( ny_of_t6_on_t6g - jend_of_t7_on_t6g )) - -# This if-statement can hopefully be removed once EMC agrees to make their -# GFDLgrid type grids (tile 7) symmetric about tile 6. - if [ ${num_bot_margin_cells_on_t6g} -ne ${num_top_margin_cells_on_t6g} ]; then - print_err_msg_exit "\ -In order for tile 7 to be centered in the y direction on tile 6, the y- -direction tile 6 cell indices at which tile 7 starts and ends (given by -jstart_of_t7_on_t6g and jend_of_t7_on_t6g, respectively) must be set -such that the number of tile 6 cells in the margin between the left -boundaries of tiles 6 and 7 (given by num_left_margin_cells_on_t6g) is -equal to that in the margin between their right boundaries (given by -num_right_margin_cells_on_t6g): - jstart_of_t7_on_t6g = ${jstart_of_t7_on_t6g} - jend_of_t7_on_t6g = ${jend_of_t7_on_t6g} - num_bot_margin_cells_on_t6g = ${num_bot_margin_cells_on_t6g} - num_top_margin_cells_on_t6g = ${num_top_margin_cells_on_t6g} -Note that the total number of cells in the y-direction on tile 6 is gi- -ven by: - ny_of_t6_on_t6g = ${ny_of_t6_on_t6g} -Please reset jstart_of_t7_on_t6g and jend_of_t7_on_t6g and rerun." - fi - - lon_of_t7_ctr="${lon_of_t6_ctr}" - lat_of_t7_ctr="${lat_of_t6_ctr}" -# -#----------------------------------------------------------------------- -# -# The grid generation script grid_gen_scr called below in turn calls the -# make_hgrid utility/executable to construct the regional grid. make_- -# hgrid accepts as arguments the index limits (i.e. starting and ending -# indices) of the regional grid on the supergrid of the regional grid's -# parent tile. The regional grid's parent tile is tile 6, and the su- -# pergrid of any given tile is defined as the grid obtained by doubling -# the number of cells in each direction on that tile's grid. We will -# denote these index limits by -# -# istart_of_t7_on_t6sg -# iend_of_t7_on_t6sg -# jstart_of_t7_on_t6sg -# jend_of_t7_on_t6sg -# -# The "_T6SG" suffix in these names is used to indicate that the indices -# are on the supergrid of tile 6. Recall, however, that we have as in- -# puts the index limits of the regional grid on the tile 6 grid, not its -# supergrid. These are given by -# -# istart_of_t7_on_t6g -# iend_of_t7_on_t6g -# jstart_of_t7_on_t6g -# jend_of_t7_on_t6g -# -# We can obtain the former from the latter by recalling that the super- -# grid has twice the resolution of the original grid. Thus, -# -# istart_of_t7_on_t6sg = 2*istart_of_t7_on_t6g - 1 -# iend_of_t7_on_t6sg = 2*iend_of_t7_on_t6g -# jstart_of_t7_on_t6sg = 2*jstart_of_t7_on_t6g - 1 -# jend_of_t7_on_t6sg = 2*jend_of_t7_on_t6g -# -# These are obtained assuming that grid cells on tile 6 must either be -# completely within the regional domain or completely outside of it, -# i.e. the boundary of the regional grid must coincide with gridlines -# on the tile 6 grid; it cannot cut through tile 6 cells. (Note that -# this implies that the starting indices on the tile 6 supergrid must be -# odd while the ending indices must be even; the above expressions sa- -# tisfy this requirement.) We perfrom these calculations next. -# -#----------------------------------------------------------------------- -# - istart_of_t7_on_t6sg=$(( 2*istart_of_t7_on_t6g - 1 )) - iend_of_t7_on_t6sg=$(( 2*iend_of_t7_on_t6g )) - jstart_of_t7_on_t6sg=$(( 2*jstart_of_t7_on_t6g - 1 )) - jend_of_t7_on_t6sg=$(( 2*jend_of_t7_on_t6g )) -# -#----------------------------------------------------------------------- -# -# If we simply pass to make_hgrid the index limits of the regional grid -# on the tile 6 supergrid calculated above, make_hgrid will generate a -# regional grid without a halo. To obtain a regional grid with a halo, -# we must pass to make_hgrid the index limits (on the tile 6 supergrid) -# of the regional grid including a halo. We will let the variables -# -# istart_of_t7_with_halo_on_t6sg -# iend_of_t7_with_halo_on_t6sg -# jstart_of_t7_with_halo_on_t6sg -# jend_of_t7_with_halo_on_t6sg -# -# denote these limits. The reason we include "_wide_halo" in these va- -# riable names is that the halo of the grid that we will first generate -# will be wider than the halos that are actually needed as inputs to the -# FV3LAM model (i.e. the 0-cell-wide, 3-cell-wide, and 4-cell-wide halos -# described above). We will generate the grids with narrower halos that -# the model needs later on by "shaving" layers of cells from this wide- -# halo grid. Next, we describe how to calculate the above indices. -# -# Let halo_width_on_t7g denote the width of the "wide" halo in units of number of -# grid cells on the regional grid (i.e. tile 7) that we'd like to have -# along all four edges of the regional domain (left, right, bottom, and -# top). To obtain the corresponding halo width in units of number of -# cells on the tile 6 grid -- which we denote by halo_width_on_t6g -- we simply di- -# vide halo_width_on_t7g by the refinement ratio, i.e. -# -# halo_width_on_t6g = halo_width_on_t7g/refine_ratio_t6g_to_t7g -# -# The corresponding halo width on the tile 6 supergrid is then given by -# -# halo_width_on_t6sg = 2*halo_width_on_t6g -# = 2*halo_width_on_t7g/refine_ratio_t6g_to_t7g -# -# Note that halo_width_on_t6sg must be an integer, but the expression for it de- -# rived above may not yield an integer. To ensure that the halo has a -# width of at least halo_width_on_t7g cells on the regional grid, we round up the -# result of the expression above for halo_width_on_t6sg, i.e. we redefine halo_width_on_t6sg -# to be -# -# halo_width_on_t6sg = ceil(2*halo_width_on_t7g/refine_ratio_t6g_to_t7g) -# -# where ceil(...) is the ceiling function, i.e. it rounds its floating -# point argument up to the next larger integer. Since in bash division -# of two integers returns a truncated integer and since bash has no -# built-in ceil(...) function, we perform the rounding-up operation by -# adding the denominator (of the argument of ceil(...) above) minus 1 to -# the original numerator, i.e. by redefining halo_width_on_t6sg to be -# -# halo_width_on_t6sg = (2*halo_width_on_t7g + refine_ratio_t6g_to_t7g - 1)/refine_ratio_t6g_to_t7g -# -# This trick works when dividing one positive integer by another. -# -# In order to calculate halo_width_on_t6g using the above expression, we must -# first specify halo_width_on_t7g. Next, we specify an initial value for it by -# setting it to one more than the largest-width halo that the model ac- -# tually needs, which is NH4. We then calculate halo_width_on_t6sg using the -# above expression. Note that these values of halo_width_on_t7g and halo_width_on_t6sg will -# likely not be their final values; their final values will be calcula- -# ted later below after calculating the starting and ending indices of -# the regional grid with wide halo on the tile 6 supergrid and then ad- -# justing the latter to satisfy certain conditions. -# -#----------------------------------------------------------------------- -# - halo_width_on_t7g=$(( NH4 + 1 )) - halo_width_on_t6sg=$(( (2*halo_width_on_t7g + refine_ratio_t6g_to_t7g - 1)/refine_ratio_t6g_to_t7g )) -# -#----------------------------------------------------------------------- -# -# With an initial value of halo_width_on_t6sg now available, we can obtain the -# tile 6 supergrid index limits of the regional domain (including the -# wide halo) from the index limits for the regional domain without a ha- -# lo by simply subtracting halo_width_on_t6sg from the lower index limits and add- -# ing halo_width_on_t6sg to the upper index limits, i.e. -# -# istart_of_t7_with_halo_on_t6sg = istart_of_t7_on_t6sg - halo_width_on_t6sg -# iend_of_t7_with_halo_on_t6sg = iend_of_t7_on_t6sg + halo_width_on_t6sg -# jstart_of_t7_with_halo_on_t6sg = jstart_of_t7_on_t6sg - halo_width_on_t6sg -# jend_of_t7_with_halo_on_t6sg = jend_of_t7_on_t6sg + halo_width_on_t6sg -# -# We calculate these next. -# -#----------------------------------------------------------------------- -# - istart_of_t7_with_halo_on_t6sg=$(( istart_of_t7_on_t6sg - halo_width_on_t6sg )) - iend_of_t7_with_halo_on_t6sg=$(( iend_of_t7_on_t6sg + halo_width_on_t6sg )) - jstart_of_t7_with_halo_on_t6sg=$(( jstart_of_t7_on_t6sg - halo_width_on_t6sg )) - jend_of_t7_with_halo_on_t6sg=$(( jend_of_t7_on_t6sg + halo_width_on_t6sg )) -# -#----------------------------------------------------------------------- -# -# As for the regional grid without a halo, the regional grid with a wide -# halo that make_hgrid will generate must be such that grid cells on -# tile 6 either lie completely within this grid or outside of it, i.e. -# they cannot lie partially within/outside of it. This implies that the -# starting indices on the tile 6 supergrid of the grid with wide halo -# must be odd while the ending indices must be even. Thus, below, we -# subtract 1 from the starting indices if they are even (which ensures -# that there will be at least halo_width_on_t7g halo cells along the left and bot- -# tom boundaries), and we add 1 to the ending indices if they are odd -# (which ensures that there will be at least halo_width_on_t7g halo cells along the -# right and top boundaries). -# -#----------------------------------------------------------------------- -# - if [ $(( istart_of_t7_with_halo_on_t6sg%2 )) -eq 0 ]; then - istart_of_t7_with_halo_on_t6sg=$(( istart_of_t7_with_halo_on_t6sg - 1 )) - fi - - if [ $(( iend_of_t7_with_halo_on_t6sg%2 )) -eq 1 ]; then - iend_of_t7_with_halo_on_t6sg=$(( iend_of_t7_with_halo_on_t6sg + 1 )) - fi - - if [ $(( jstart_of_t7_with_halo_on_t6sg%2 )) -eq 0 ]; then - jstart_of_t7_with_halo_on_t6sg=$(( jstart_of_t7_with_halo_on_t6sg - 1 )) - fi - - if [ $(( jend_of_t7_with_halo_on_t6sg%2 )) -eq 1 ]; then - jend_of_t7_with_halo_on_t6sg=$(( jend_of_t7_with_halo_on_t6sg + 1 )) - fi -# -#----------------------------------------------------------------------- -# -# Save the current shell options and temporarily turn off the xtrace op- -# tion to prevent clutter in stdout. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Now that the starting and ending tile 6 supergrid indices of the re- -# gional grid with the wide halo have been calculated (and adjusted), we -# recalculate the width of the wide halo on: -# -# 1) the tile 6 supergrid; -# 2) the tile 6 grid; and -# 3) the tile 7 grid. -# -# These are the final values of these quantities that are guaranteed to -# correspond to the starting and ending indices on the tile 6 supergrid. -# -#----------------------------------------------------------------------- -# - print_info_msg "$verbose" " -Original values of the halo width on the tile 6 supergrid and on the -tile 7 grid are: - halo_width_on_t6sg = ${halo_width_on_t6sg} - halo_width_on_t7g = ${halo_width_on_t7g}" - - halo_width_on_t6sg=$(( istart_of_t7_on_t6sg - istart_of_t7_with_halo_on_t6sg )) - halo_width_on_t6g=$(( halo_width_on_t6sg/2 )) - halo_width_on_t7g=$(( halo_width_on_t6g*refine_ratio_t6g_to_t7g )) - - print_info_msg "$verbose" " -Values of the halo width on the tile 6 supergrid and on the tile 7 grid -AFTER adjustments are: - halo_width_on_t6sg = ${halo_width_on_t6sg} - halo_width_on_t7g = ${halo_width_on_t7g}" -# -#----------------------------------------------------------------------- -# -# Calculate the number of cells that the regional domain (without halo) -# has in each of the two horizontal directions (say x and y). We denote -# these by nx_of_t7_on_t7g and ny_of_t7_on_t7g, respectively. These -# will be needed in the "shave" steps in the grid generation task of the -# workflow. -# -#----------------------------------------------------------------------- -# - nx_of_t7_on_t6sg=$(( iend_of_t7_on_t6sg - istart_of_t7_on_t6sg + 1 )) - nx_of_t7_on_t6g=$(( nx_of_t7_on_t6sg/2 )) - nx_of_t7_on_t7g=$(( nx_of_t7_on_t6g*refine_ratio_t6g_to_t7g )) - - ny_of_t7_on_t6sg=$(( jend_of_t7_on_t6sg - jstart_of_t7_on_t6sg + 1 )) - ny_of_t7_on_t6g=$(( ny_of_t7_on_t6sg/2 )) - ny_of_t7_on_t7g=$(( ny_of_t7_on_t6g*refine_ratio_t6g_to_t7g )) -# -# The following are set only for informational purposes. -# - nx_of_t6_on_t6sg=$(( 2*nx_of_t6_on_t6g )) - ny_of_t6_on_t6sg=$(( 2*ny_of_t6_on_t6g )) - - prime_factors_nx_of_t7_on_t7g=$( factor ${nx_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) - prime_factors_ny_of_t7_on_t7g=$( factor ${ny_of_t7_on_t7g} | $SED -r -e 's/^[0-9]+: (.*)/\1/' ) - - print_info_msg "$verbose" " -The number of cells in the two horizontal directions (x and y) on the -parent tile's (tile 6) grid and supergrid are: - nx_of_t6_on_t6g = ${nx_of_t6_on_t6g} - ny_of_t6_on_t6g = ${ny_of_t6_on_t6g} - nx_of_t6_on_t6sg = ${nx_of_t6_on_t6sg} - ny_of_t6_on_t6sg = ${ny_of_t6_on_t6sg} - -The number of cells in the two horizontal directions on the tile 6 grid -and supergrid that the regional domain (tile 7) WITHOUT A HALO encompas- -ses are: - nx_of_t7_on_t6g = ${nx_of_t7_on_t6g} - ny_of_t7_on_t6g = ${ny_of_t7_on_t6g} - nx_of_t7_on_t6sg = ${nx_of_t7_on_t6sg} - ny_of_t7_on_t6sg = ${ny_of_t7_on_t6sg} - -The starting and ending i and j indices on the tile 6 grid used to gene- -rate this regional grid are: - istart_of_t7_on_t6g = ${istart_of_t7_on_t6g} - iend_of_t7_on_t6g = ${iend_of_t7_on_t6g} - jstart_of_t7_on_t6g = ${jstart_of_t7_on_t6g} - jend_of_t7_on_t6g = ${jend_of_t7_on_t6g} - -The corresponding starting and ending i and j indices on the tile 6 su- -pergrid are: - istart_of_t7_on_t6sg = ${istart_of_t7_on_t6sg} - iend_of_t7_on_t6sg = ${iend_of_t7_on_t6sg} - jstart_of_t7_on_t6sg = ${jstart_of_t7_on_t6sg} - jend_of_t7_on_t6sg = ${jend_of_t7_on_t6sg} - -The refinement ratio (ratio of the number of cells in tile 7 that abut -a single cell in tile 6) is: - refine_ratio_t6g_to_t7g = ${refine_ratio_t6g_to_t7g} - -The number of cells in the two horizontal directions on the regional do- -main's (i.e. tile 7's) grid WITHOUT A HALO are: - nx_of_t7_on_t7g = ${nx_of_t7_on_t7g} - ny_of_t7_on_t7g = ${ny_of_t7_on_t7g} - -The prime factors of nx_of_t7_on_t7g and ny_of_t7_on_t7g are (useful for -determining an MPI task layout): - prime_factors_nx_of_t7_on_t7g: ${prime_factors_nx_of_t7_on_t7g} - prime_factors_ny_of_t7_on_t7g: ${prime_factors_ny_of_t7_on_t7g}" -# -#----------------------------------------------------------------------- -# -# For informational purposes, calculate the number of cells in each di- -# rection on the regional grid including the wide halo (of width halo_- -# width_on_t7g cells). We denote these by nx_of_t7_with_halo_on_t7g and -# ny_of_t7_with_halo_on_t7g, respectively. -# -#----------------------------------------------------------------------- -# - nx_of_t7_with_halo_on_t6sg=$(( iend_of_t7_with_halo_on_t6sg - istart_of_t7_with_halo_on_t6sg + 1 )) - nx_of_t7_with_halo_on_t6g=$(( nx_of_t7_with_halo_on_t6sg/2 )) - nx_of_t7_with_halo_on_t7g=$(( nx_of_t7_with_halo_on_t6g*refine_ratio_t6g_to_t7g )) - - ny_of_t7_with_halo_on_t6sg=$(( jend_of_t7_with_halo_on_t6sg - jstart_of_t7_with_halo_on_t6sg + 1 )) - ny_of_t7_with_halo_on_t6g=$(( ny_of_t7_with_halo_on_t6sg/2 )) - ny_of_t7_with_halo_on_t7g=$(( ny_of_t7_with_halo_on_t6g*refine_ratio_t6g_to_t7g )) - - print_info_msg "$verbose" " -nx_of_t7_with_halo_on_t7g = ${nx_of_t7_with_halo_on_t7g} \ -(istart_of_t7_with_halo_on_t6sg = ${istart_of_t7_with_halo_on_t6sg}, \ -iend_of_t7_with_halo_on_t6sg = ${iend_of_t7_with_halo_on_t6sg})" - - print_info_msg "$verbose" " -ny_of_t7_with_halo_on_t7g = ${ny_of_t7_with_halo_on_t7g} \ -(jstart_of_t7_with_halo_on_t6sg = ${jstart_of_t7_with_halo_on_t6sg}, \ -jend_of_t7_with_halo_on_t6sg = ${jend_of_t7_with_halo_on_t6sg})" -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_lon_of_t7_ctr}" ]; then - printf -v ${outvarname_lon_of_t7_ctr} "%s" "${lon_of_t7_ctr}" - fi - - if [ ! -z "${outvarname_lat_of_t7_ctr}" ]; then - printf -v ${outvarname_lat_of_t7_ctr} "%s" "${lat_of_t7_ctr}" - fi - - if [ ! -z "${outvarname_nx_of_t7_on_t7g}" ]; then - printf -v ${outvarname_nx_of_t7_on_t7g} "%s" "${nx_of_t7_on_t7g}" - fi - - if [ ! -z "${outvarname_ny_of_t7_on_t7g}" ]; then - printf -v ${outvarname_ny_of_t7_on_t7g} "%s" "${ny_of_t7_on_t7g}" - fi - - if [ ! -z "${outvarname_halo_width_on_t7g}" ]; then - printf -v ${outvarname_halo_width_on_t7g} "%s" "${halo_width_on_t7g}" - fi - - if [ ! -z "${outvarname_stretch_factor}" ]; then - printf -v ${outvarname_stretch_factor} "%s" "${stretch_factor}" - fi - - if [ ! -z "${outvarname_istart_of_t7_with_halo_on_t6sg}" ]; then - printf -v ${outvarname_istart_of_t7_with_halo_on_t6sg} "%s" "${istart_of_t7_with_halo_on_t6sg}" - fi - - if [ ! -z "${outvarname_iend_of_t7_with_halo_on_t6sg}" ]; then - printf -v ${outvarname_iend_of_t7_with_halo_on_t6sg} "%s" "${iend_of_t7_with_halo_on_t6sg}" - fi - - if [ ! -z "${outvarname_jstart_of_t7_with_halo_on_t6sg}" ]; then - printf -v ${outvarname_jstart_of_t7_with_halo_on_t6sg} "%s" "${jstart_of_t7_with_halo_on_t6sg}" - fi - - if [ ! -z "${outvarname_jend_of_t7_with_halo_on_t6sg}" ]; then - printf -v ${outvarname_jend_of_t7_with_halo_on_t6sg} "%s" "${jend_of_t7_with_halo_on_t6sg}" - fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options before turning off xtrace. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh deleted file mode 100644 index bd6f5fd377..0000000000 --- a/ush/set_ozone_param.sh +++ /dev/null @@ -1,241 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that: -# -# (1) Determines the ozone parameterization being used by checking in the -# CCPP physics suite XML. -# -# (2) Sets the name of the global ozone production/loss file in the FIXgsm -# FIXgsm system directory to copy to the experiment's FIXam directory. -# -# (3) Resets the last element of the workflow array variable -# FIXgsm_FILES_TO_COPY_TO_FIXam that contains the files to copy from -# FIXgsm to FIXam (this last element is initially set to a dummy -# value) to the name of the ozone production/loss file set in the -# previous step. -# -# (4) Resets the element of the workflow array variable -# CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING (this array contains the -# mapping between the symlinks to create in any cycle directory and -# the files in the FIXam directory that are their targets) that -# specifies the mapping for the ozone symlink/file such that the -# target FIXam file name is set to the name of the ozone production/ -# loss file set above. -# -#----------------------------------------------------------------------- -# -function set_ozone_param() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ -"ccpp_phys_suite_fp" \ -"output_varname_ozone_param" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local ozone_param \ - regex_search \ - fixgsm_ozone_fn \ - i \ - ozone_symlink \ - fixgsm_ozone_fn_is_set \ - regex_search \ - num_symlinks \ - mapping \ - symlink \ - mapping_ozone \ - msg -# -#----------------------------------------------------------------------- -# -# Get the name of the ozone parameterization being used. There are two -# possible ozone parameterizations: -# -# (1) A parameterization developed/published in 2015. Here, we refer to -# this as the 2015 parameterization. If this is being used, then we -# set the variable ozone_param to the string "ozphys_2015". -# -# (2) A parameterization developed/published sometime after 2015. Here, -# we refer to this as the after-2015 parameterization. If this is -# being used, then we set the variable ozone_param to the string -# "ozphys". -# -# We check the CCPP physics suite definition file (SDF) to determine the -# parameterization being used. If this file contains the line -# -# ozphys_2015 -# -# then the 2015 parameterization is being used. If it instead contains -# the line -# -# ozphys -# -# then the after-2015 parameterization is being used. (The SDF should -# contain exactly one of these lines; not both nor neither; we check for -# this.) -# -#----------------------------------------------------------------------- -# - regex_search="^[ ]*(ozphys.*)<\/scheme>[ ]*$" - ozone_param=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) - - if [ "${ozone_param}" = "ozphys_2015" ]; then - fixgsm_ozone_fn="ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77" - elif [ "${ozone_param}" = "ozphys" ]; then - fixgsm_ozone_fn="global_o3prdlos.f77" - else - print_err_msg_exit "\ -Unknown ozone parameterization (ozone_param) or no ozone parameterization -specified in the CCPP physics suite file (ccpp_phys_suite_fp): - ccpp_phys_suite_fp = \"${ccpp_phys_suite_fp}\" - ozone_param = \"${ozone_param}\"" - fi -# -#----------------------------------------------------------------------- -# -# Set the last element of the array FIXgsm_FILES_TO_COPY_TO_FIXam to the -# name of the ozone production/loss file to copy from the FIXgsm to the -# FIXam directory. -# -#----------------------------------------------------------------------- -# -i=$(( ${#FIXgsm_FILES_TO_COPY_TO_FIXam[@]} - 1 )) -FIXgsm_FILES_TO_COPY_TO_FIXam[$i]="${fixgsm_ozone_fn}" -# -#----------------------------------------------------------------------- -# -# Set the element in the array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING that -# specifies the mapping between the symlink for the ozone production/loss -# file that must be created in each cycle directory and its target in the -# FIXam directory. The name of the symlink is alrady in the array, but -# the target is not because it depends on the ozone parameterization that -# the physics suite uses. Since we determined the ozone parameterization -# above, we now set the target of the symlink accordingly. -# -#----------------------------------------------------------------------- -# -ozone_symlink="global_o3prdlos.f77" -fixgsm_ozone_fn_is_set="FALSE" -regex_search="^[ ]*([^| ]*)[ ]*[|][ ]*([^| ]*)[ ]*$" -num_symlinks=${#CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]} - -for (( i=0; i<${num_symlinks}; i++ )); do - mapping="${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]}" - symlink=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\1/p" ) - if [ "$symlink" = "${ozone_symlink}" ]; then - regex_search="^[ ]*([^| ]+[ ]*)[|][ ]*([^| ]*)[ ]*$" - mapping_ozone=$( printf "%s\n" "$mapping" | \ - $SED -n -r -e "s/${regex_search}/\1/p" ) - mapping_ozone="${mapping_ozone}| ${fixgsm_ozone_fn}" - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[$i]="${mapping_ozone}" - fixgsm_ozone_fn_is_set="TRUE" - break - fi -done -# -#----------------------------------------------------------------------- -# -# If fixgsm_ozone_fn_is_set is set to "TRUE", then the appropriate element -# of the array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING was set successfully. -# In this case, print out the new version of this array. Otherwise, print -# out an error message and exit. -# -#----------------------------------------------------------------------- -# -if [ "${fixgsm_ozone_fn_is_set}" = "TRUE" ]; then - - msg=" -After setting the file name of the ozone production/loss file in the -FIXgsm directory (based on the ozone parameterization specified in the -CCPP suite definition file), the array specifying the mapping between -the symlinks that need to be created in the cycle directories and the -files in the FIXam directory is: - - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = ( \\ -" - msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) - msg="$msg"$( printf "\n)" ) - print_info_msg "$VERBOSE" "$msg" - -else - - print_err_msg_exit "\ -Unable to set name of the ozone production/loss file in the FIXgsm directory -in the array that specifies the mapping between the symlinks that need to -be created in the cycle directories and the files in the FIXgsm directory: - fixgsm_ozone_fn_is_set = \"${fixgsm_ozone_fn_is_set}\"" - -fi -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - eval ${output_varname_ozone_param}="${ozone_param}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index 2838517150..970187e1de 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -3,8 +3,6 @@ import unittest import os -from constants import radius_Earth,degs_per_radian - from python_utils import process_args,import_vars,export_vars,set_env_var,get_env_var,\ print_input_args,define_macos_utilities, load_config_file, \ cfg_to_yaml_str @@ -18,9 +16,11 @@ def set_predef_grid_params(): None """ # import all environement variables - import_vars() + IMPORTS = ['PREDEF_GRID_NAME', 'QUILTING', 'DT_ATMOS', 'LAYOUT_X', 'LAYOUT_Y', 'BLOCKSIZE'] + import_vars(env_vars=IMPORTS) - params_dict = load_config_file("predef_grid_params.yaml") + USHDIR = os.path.dirname(os.path.abspath(__file__)) + params_dict = load_config_file(os.path.join(USHDIR,"predef_grid_params.yaml")) params_dict = params_dict[PREDEF_GRID_NAME] # if QUILTING = False, skip variables that start with "WRTCMP_" @@ -34,17 +34,14 @@ def set_predef_grid_params(): if globals()[var] is not None: params_dict[var] = globals()[var] - #export variables to environment + # export variables to environment export_vars(source_dict=params_dict) -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# + + return params_dict + if __name__ == "__main__": - set_predef_grid_params() + params_dict = set_predef_grid_params() + print( cfg_to_shell_str(params_dict), end='' ) class Testing(unittest.TestCase): def test_set_predef_grid_params(self): diff --git a/ush/set_predef_grid_params.sh b/ush/set_predef_grid_params.sh deleted file mode 100644 index e0c89644df..0000000000 --- a/ush/set_predef_grid_params.sh +++ /dev/null @@ -1,1765 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets grid parameters -# for the specified predefined grid. -# -#----------------------------------------------------------------------- -# -function set_predef_grid_params() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set directories. -# -#----------------------------------------------------------------------- -# - local homerrfs=${scrfunc_dir%/*} - local ushdir="$homerrfs/ush" -# -#----------------------------------------------------------------------- -# -# Source the file containing various mathematical, physical, etc constants. -# -#----------------------------------------------------------------------- -# - . $ushdir/constants.sh -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. -# Then process the arguments provided to this script/function (which -# should consist of a set of name-value pairs of the form arg1="value1", -# etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "predef_grid_name" \ - "dt_atmos" \ - "layout_x" \ - "layout_y" \ - "blocksize" \ - "quilting" \ - "outvarname_grid_gen_method" \ - "outvarname_esggrid_lon_ctr" \ - "outvarname_esggrid_lat_ctr" \ - "outvarname_esggrid_delx" \ - "outvarname_esggrid_dely" \ - "outvarname_esggrid_nx" \ - "outvarname_esggrid_ny" \ - "outvarname_esggrid_pazi" \ - "outvarname_esggrid_wide_halo_width" \ - "outvarname_gfdlgrid_lon_t6_ctr" \ - "outvarname_gfdlgrid_lat_t6_ctr" \ - "outvarname_gfdlgrid_stretch_fac" \ - "outvarname_gfdlgrid_num_cells" \ - "outvarname_gfdlgrid_refine_ratio" \ - "outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g" \ - "outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g" \ - "outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g" \ - "outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g" \ - "outvarname_gfdlgrid_use_num_cells_in_filenames" \ - "outvarname_dt_atmos" \ - "outvarname_layout_x" \ - "outvarname_layout_y" \ - "outvarname_blocksize" \ - "outvarname_wrtcmp_write_groups" \ - "outvarname_wrtcmp_write_tasks_per_group" \ - "outvarname_wrtcmp_output_grid" \ - "outvarname_wrtcmp_cen_lon" \ - "outvarname_wrtcmp_cen_lat" \ - "outvarname_wrtcmp_stdlat1" \ - "outvarname_wrtcmp_stdlat2" \ - "outvarname_wrtcmp_nx" \ - "outvarname_wrtcmp_ny" \ - "outvarname_wrtcmp_lon_lwr_left" \ - "outvarname_wrtcmp_lat_lwr_left" \ - "outvarname_wrtcmp_lon_upr_rght" \ - "outvarname_wrtcmp_lat_upr_rght" \ - "outvarname_wrtcmp_dx" \ - "outvarname_wrtcmp_dy" \ - "outvarname_wrtcmp_dlon" \ - "outvarname_wrtcmp_dlat" \ - ) - process_args "valid_args" "$@" -# -#----------------------------------------------------------------------- -# -# Declare and initialize local variables. -# -#----------------------------------------------------------------------- -# - local __grid_gen_method__="" \ - __esggrid_lon_ctr__="" \ - __esggrid_lat_ctr__="" \ - __esggrid_delx__="" \ - __esggrid_dely__="" \ - __esggrid_nx__="" \ - __esggrid_ny__="" \ - __esggrid_pazi__="" \ - __esggrid_wide_halo_width__="" \ - __gfdlgrid_lon_t6_ctr__="" \ - __gfdlgrid_lat_t6_ctr__="" \ - __gfdlgrid_stretch_fac__="" \ - __gfdlgrid_num_cells__="" \ - __gfdlgrid_refine_ratio__="" \ - __gfdlgrid_istart_of_rgnl_dom_on_t6g__="" \ - __gfdlgrid_iend_of_rgnl_dom_on_t6g__="" \ - __gfdlgrid_jstart_of_rgnl_dom_on_t6g__="" \ - __gfdlgrid_jend_of_rgnl_dom_on_t6g__="" \ - __gfdlgrid_use_num_cells_in_filenames__="" \ - __dt_atmos__="" \ - __layout_x__="" \ - __layout_y__="" \ - __blocksize__="" \ - __wrtcmp_write_groups__="" \ - __wrtcmp_write_tasks_per_group__="" \ - __wrtcmp_output_grid__="" \ - __wrtcmp_cen_lon__="" \ - __wrtcmp_cen_lat__="" \ - __wrtcmp_stdlat1__="" \ - __wrtcmp_stdlat2__="" \ - __wrtcmp_nx__="" \ - __wrtcmp_ny__="" \ - __wrtcmp_lon_lwr_left__="" \ - __wrtcmp_lat_lwr_left__="" \ - __wrtcmp_lon_upr_rght__="" \ - __wrtcmp_lat_upr_rght__="" \ - __wrtcmp_dx__="" \ - __wrtcmp_dy__="" \ - __wrtcmp_dlon__="" \ - __wrtcmp_dlat__="" \ - num_margin_cells_T6_left="" \ - num_margin_cells_T6_right="" \ - num_margin_cells_T6_bottom="" \ - num_margin_cells_T6_top="" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# -# print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Make sure that the input argument "quilting" is set to a valid value. -# -#----------------------------------------------------------------------- -# - check_var_valid_value "quilting" "valid_vals_BOOLEAN" - quilting=$(boolify "$quilting") -# -#----------------------------------------------------------------------- -# -# Set grid and other parameters according to the value of the predefined -# domain (predef_grid_name). Note that the code will enter this script -# only if predef_grid_name has a valid (and non-empty) value. -# -#################### -# The following comments need to be updated: -#################### -# -# 1) Reset the experiment title (expt_title). -# 2) Reset the grid parameters. -# 3) If the write component is to be used (i.e. "quilting" is set to -# "TRUE") and the variable WRTCMP_PARAMS_TMPL_FN containing the name -# of the write-component template file is unset or empty, set that -# filename variable to the appropriate preexisting template file. -# -# For the predefined domains, we determine the starting and ending indi- -# ces of the regional grid within tile 6 by specifying margins (in units -# of number of cells on tile 6) between the boundary of tile 6 and that -# of the regional grid (tile 7) along the left, right, bottom, and top -# portions of these boundaries. Note that we do not use "west", "east", -# "south", and "north" here because the tiles aren't necessarily orient- -# ed such that the left boundary segment corresponds to the west edge, -# etc. The widths of these margins (in units of number of cells on tile -# 6) are specified via the parameters -# -# num_margin_cells_T6_left -# num_margin_cells_T6_right -# num_margin_cells_T6_bottom -# num_margin_cells_T6_top -# -# where the "_T6" in these names is used to indicate that the cell count -# is on tile 6, not tile 7. -# -# Note that we must make the margins wide enough (by making the above -# four parameters large enough) such that a region of halo cells around -# the boundary of the regional grid fits into the margins, i.e. such -# that the halo does not overrun the boundary of tile 6. (The halo is -# added later in another script; its function is to feed in boundary -# conditions to the regional grid.) Currently, a halo of 5 regional -# grid cells is used around the regional grid. Setting num_margin_- -# cells_T6_... to at least 10 leaves enough room for this halo. -# -#----------------------------------------------------------------------- -# - case "${predef_grid_name}" in -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~25km cells. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUS_25km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="25000.0" - __esggrid_dely__="25000.0" - - __esggrid_nx__="219" - __esggrid_ny__="131" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-5}" - __layout_y__="${layout_y:-2}" - __blocksize__="${blocksize:-40}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="2" - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="217" - __wrtcmp_ny__="128" - __wrtcmp_lon_lwr_left__="-122.719528" - __wrtcmp_lat_lwr_left__="21.138123" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~25km cells that can be initialized from -# the HRRR. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUScompact_25km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="25000.0" - __esggrid_dely__="25000.0" - - __esggrid_nx__="202" - __esggrid_ny__="116" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-5}" - __layout_y__="${layout_y:-2}" - __blocksize__="${blocksize:-40}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="2" - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="199" - __wrtcmp_ny__="111" - __wrtcmp_lon_lwr_left__="-121.23349066" - __wrtcmp_lat_lwr_left__="23.41731593" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~13km cells. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUS_13km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="13000.0" - __esggrid_dely__="13000.0" - - __esggrid_nx__="420" - __esggrid_ny__="252" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-45}" - - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-10}" - __blocksize__="${blocksize:-32}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="416" - __wrtcmp_ny__="245" - __wrtcmp_lon_lwr_left__="-122.719528" - __wrtcmp_lat_lwr_left__="21.138123" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~13km cells that can be initialized from the HRRR. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUScompact_13km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="13000.0" - __esggrid_dely__="13000.0" - - __esggrid_nx__="396" - __esggrid_ny__="232" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-45}" - - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-10}" - __blocksize__="${blocksize:-32}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="393" - __wrtcmp_ny__="225" - __wrtcmp_lon_lwr_left__="-121.70231097" - __wrtcmp_lat_lwr_left__="22.57417972" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~3km cells. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUS_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="1820" - __esggrid_ny__="1092" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-36}" - - __layout_x__="${layout_x:-28}" - __layout_y__="${layout_y:-28}" - __blocksize__="${blocksize:-29}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="1799" - __wrtcmp_ny__="1059" - __wrtcmp_lon_lwr_left__="-122.719528" - __wrtcmp_lat_lwr_left__="21.138123" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS CONUS domain with ~3km cells that can be initialized from -# the HRRR. -# -#----------------------------------------------------------------------- -# - "RRFS_CONUScompact_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="1748" - __esggrid_ny__="1038" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-30}" - __layout_y__="${layout_y:-16}" - __blocksize__="${blocksize:-32}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="1746" - __wrtcmp_ny__="1014" - __wrtcmp_lon_lwr_left__="-122.17364391" - __wrtcmp_lat_lwr_left__="21.88588562" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS SUBCONUS domain with ~3km cells. -# -#----------------------------------------------------------------------- -# - "RRFS_SUBCONUS_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="35.0" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="840" - __esggrid_ny__="600" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-30}" - __layout_y__="${layout_y:-24}" - __blocksize__="${blocksize:-35}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="837" - __wrtcmp_ny__="595" - __wrtcmp_lon_lwr_left__="-109.97410429" - __wrtcmp_lat_lwr_left__="26.31459843" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# A subconus domain over Indianapolis, Indiana with ~3km cells. This is -# mostly for testing on a 3km grid with a much small number of cells than -# on the full CONUS. -# -#----------------------------------------------------------------------- -# - "SUBCONUS_Ind_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-86.16" - __esggrid_lat_ctr__="39.77" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="200" - __esggrid_ny__="200" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-5}" - __layout_y__="${layout_y:-5}" - __blocksize__="${blocksize:-40}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="197" - __wrtcmp_ny__="197" - __wrtcmp_lon_lwr_left__="-89.47120417" - __wrtcmp_lat_lwr_left__="37.07809642" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS Alaska domain with ~13km cells. -# -# Note: -# This grid has not been thoroughly tested (as of 20201027). -# -#----------------------------------------------------------------------- -# - "RRFS_AK_13km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-161.5" - __esggrid_lat_ctr__="63.0" - - __esggrid_delx__="13000.0" - __esggrid_dely__="13000.0" - - __esggrid_nx__="320" - __esggrid_ny__="240" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - -# __dt_atmos__="${dt_atmos:-50}" - __dt_atmos__="${dt_atmos:-10}" - - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-12}" - __blocksize__="${blocksize:-40}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - -# The following works. The numbers were obtained using the NCL scripts -# but only after manually modifying the longitutes of two of the four -# corners of the domain to add 360.0 to them. Need to automate that -# procedure. - __wrtcmp_nx__="318" - __wrtcmp_ny__="234" -# __wrtcmp_lon_lwr_left__="-187.76660836" - __wrtcmp_lon_lwr_left__="172.23339164" - __wrtcmp_lat_lwr_left__="45.77691870" - - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - -# The following rotated_latlon coordinate system parameters were obtained -# using the NCL code and work. -# if [ "$quilting" = "TRUE" ]; then -# __wrtcmp_write_groups__="1" -# __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) -# __wrtcmp_output_grid__="rotated_latlon" -# __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" -# __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" -# __wrtcmp_lon_lwr_left__="-18.47206579" -# __wrtcmp_lat_lwr_left__="-13.56176982" -# __wrtcmp_lon_upr_rght__="18.47206579" -# __wrtcmp_lat_upr_rght__="13.56176982" -## __wrtcmp_dlon__="0.11691181" -## __wrtcmp_dlat__="0.11691181" -# __wrtcmp_dlon__=$( printf "%.9f" $( bc -l <<< "(${esggrid_delx}/${radius_Earth})*${degs_per_radian}" ) ) -# __wrtcmp_dlat__=$( printf "%.9f" $( bc -l <<< "(${esggrid_dely}/${radius_Earth})*${degs_per_radian}" ) ) -# fi - ;; -# -#----------------------------------------------------------------------- -# -# The RRFS Alaska domain with ~3km cells. -# -# Note: -# This grid has not been thoroughly tested (as of 20201027). -# -#----------------------------------------------------------------------- -# - "RRFS_AK_3km") - -# if [ "${grid_gen_method}" = "GFDLgrid" ]; then -# -# __gfdlgrid_lon_t6_ctr__="-160.8" -# __gfdlgrid_lat_t6_ctr__="63.0" -# __gfdlgrid_stretch_fac__="1.161" -# __gfdlgrid_num_cells__="768" -# __gfdlgrid_refine_ratio__="4" -# -# num_margin_cells_T6_left="204" -# __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) -# -# num_margin_cells_T6_right="204" -# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) -# -# num_margin_cells_T6_bottom="249" -# __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) -# -# num_margin_cells_T6_top="249" -# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) -# -# __gfdlgrid_use_num_cells_in_filenames__="FALSE" -# -# __dt_atmos__="${dt_atmos:-18}" -# -# __layout_x__="${layout_x:-24}" -# __layout_y__="${layout_y:-24}" -# __blocksize__="${blocksize:-15}" -# -# if [ "$quilting" = "TRUE" ]; then -# __wrtcmp_write_groups__="1" -# __wrtcmp_write_tasks_per_group__="2" -# __wrtcmp_output_grid__="lambert_conformal" -# __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" -# __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" -# __wrtcmp_stdlat1__="${__gfdlgrid_lat_t6_ctr__}" -# __wrtcmp_stdlat2__="${__gfdlgrid_lat_t6_ctr__}" -# __wrtcmp_nx__="1320" -# __wrtcmp_ny__="950" -# __wrtcmp_lon_lwr_left__="173.734" -# __wrtcmp_lat_lwr_left__="46.740347" -# __wrtcmp_dx__="3000.0" -# __wrtcmp_dy__="3000.0" -# fi -# -# elif [ "${grid_gen_method}" = "ESGgrid" ]; then - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-161.5" - __esggrid_lat_ctr__="63.0" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="1380" - __esggrid_ny__="1020" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - -# __dt_atmos__="${dt_atmos:-50}" - __dt_atmos__="${dt_atmos:-10}" - - __layout_x__="${layout_x:-30}" - __layout_y__="${layout_y:-17}" - __blocksize__="${blocksize:-40}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="1379" - __wrtcmp_ny__="1003" - __wrtcmp_lon_lwr_left__="-187.89737923" - __wrtcmp_lat_lwr_left__="45.84576053" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# The WoFS domain with ~3km cells. -# -# Note: -# The WoFS domain will generate a 301 x 301 output grid (WRITE COMPONENT) and -# will eventually be movable (esggrid_lon_ctr/esggrid_lat_ctr). A python script -# python_utils/fv3write_parms_lambert will be useful to determine -# wrtcmp_lon_lwr_left and wrtcmp_lat_lwr_left locations (only for Lambert map -# projection currently) of the quilting output when the domain location is -# moved. Later, it should be integrated into the workflow. -# -#----------------------------------------------------------------------- -# - "WoFS_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-97.5" - __esggrid_lat_ctr__="38.5" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="361" - __esggrid_ny__="361" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-20}" - - __layout_x__="${layout_x:-18}" - __layout_y__="${layout_y:-12}" - __blocksize__="${blocksize:-30}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="301" - __wrtcmp_ny__="301" - __wrtcmp_lon_lwr_left__="-102.3802487" - __wrtcmp_lat_lwr_left__="34.3407918" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# A CONUS domain of GFDLgrid type with ~25km cells. -# -# Note: -# This grid is larger than the HRRRX domain and thus cannot be initialized -# using the HRRRX. -# -#----------------------------------------------------------------------- -# - "CONUS_25km_GFDLgrid") - - __grid_gen_method__="GFDLgrid" - - __gfdlgrid_lon_t6_ctr__="-97.5" - __gfdlgrid_lat_t6_ctr__="38.5" - __gfdlgrid_stretch_fac__="1.4" - __gfdlgrid_num_cells__="96" - __gfdlgrid_refine_ratio__="3" - - num_margin_cells_T6_left="12" - __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) - - num_margin_cells_T6_right="12" - __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) - - num_margin_cells_T6_bottom="16" - __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) - - num_margin_cells_T6_top="16" - __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) - - __gfdlgrid_use_num_cells_in_filenames__="TRUE" - - __dt_atmos__="${dt_atmos:-225}" - - __layout_x__="${layout_x:-6}" - __layout_y__="${layout_y:-4}" - __blocksize__="${blocksize:-36}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="rotated_latlon" - __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" - __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" - __wrtcmp_lon_lwr_left__="-24.40085141" - __wrtcmp_lat_lwr_left__="-19.65624142" - __wrtcmp_lon_upr_rght__="24.40085141" - __wrtcmp_lat_upr_rght__="19.65624142" - __wrtcmp_dlon__="0.22593381" - __wrtcmp_dlat__="0.22593381" - fi - ;; -# -#----------------------------------------------------------------------- -# -# A CONUS domain of GFDLgrid type with ~3km cells. -# -# Note: -# This grid is larger than the HRRRX domain and thus cannot be initialized -# using the HRRRX. -# -#----------------------------------------------------------------------- -# - "CONUS_3km_GFDLgrid") - - __grid_gen_method__="GFDLgrid" - - __gfdlgrid_lon_t6_ctr__="-97.5" - __gfdlgrid_lat_t6_ctr__="38.5" - __gfdlgrid_stretch_fac__="1.5" - __gfdlgrid_num_cells__="768" - __gfdlgrid_refine_ratio__="3" - - num_margin_cells_T6_left="69" - __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) - - num_margin_cells_T6_right="69" - __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) - - num_margin_cells_T6_bottom="164" - __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) - - num_margin_cells_T6_top="164" - __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) - - __gfdlgrid_use_num_cells_in_filenames__="TRUE" - - __dt_atmos__="${dt_atmos:-18}" - - __layout_x__="${layout_x:-30}" - __layout_y__="${layout_y:-22}" - __blocksize__="${blocksize:-35}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__=$(( 1*__layout_y__ )) - __wrtcmp_output_grid__="rotated_latlon" - __wrtcmp_cen_lon__="${__gfdlgrid_lon_t6_ctr__}" - __wrtcmp_cen_lat__="${__gfdlgrid_lat_t6_ctr__}" - __wrtcmp_lon_lwr_left__="-25.23144805" - __wrtcmp_lat_lwr_left__="-15.82130419" - __wrtcmp_lon_upr_rght__="25.23144805" - __wrtcmp_lat_upr_rght__="15.82130419" - __wrtcmp_dlon__="0.02665763" - __wrtcmp_dlat__="0.02665763" - fi - ;; -# -#----------------------------------------------------------------------- -# -# EMC's Alaska grid. -# -#----------------------------------------------------------------------- -# - "EMC_AK") - -# if [ "${grid_gen_method}" = "GFDLgrid" ]; then - -# Values from an EMC script. - -### rocoto items -# -#fcstnodes=68 -#bcnodes=11 -#postnodes=2 -#goespostnodes=5 -#goespostthrottle=6 -#sh=06 -#eh=18 -# -### namelist items -# -#task_layout_x=16 -#task_layout_y=48 -#npx=1345 -#npy=1153 -#target_lat=61.0 -#target_lon=-153.0 -# -### model config items -# -#write_groups=2 -#write_tasks_per_group=24 -#cen_lon=$target_lon -#cen_lat=$target_lat -#lon1=-18.0 -#lat1=-14.79 -#lon2=18.0 -#lat2=14.79 -#dlon=0.03 -#dlat=0.03 - -# __gfdlgrid_lon_t6_ctr__="-153.0" -# __gfdlgrid_lat_t6_ctr__="61.0" -# __gfdlgrid_stretch_fac__="1.0" # ??? -# __gfdlgrid_num_cells__="768" -# __gfdlgrid_refine_ratio__="3" # ??? -# -# num_margin_cells_T6_left="61" -# __gfdlgrid_istart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_left + 1 )) -# -# num_margin_cells_T6_right="67" -# __gfdlgrid_iend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_right )) -# -# num_margin_cells_T6_bottom="165" -# __gfdlgrid_jstart_of_rgnl_dom_on_t6g__=$(( num_margin_cells_T6_bottom + 1 )) -# -# num_margin_cells_T6_top="171" -# __gfdlgrid_jend_of_rgnl_dom_on_t6g__=$(( __gfdlgrid_num_cells__ - num_margin_cells_T6_top )) -# -# __gfdlgrid_use_num_cells_in_filenames__="TRUE" -# -# __dt_atmos__="${dt_atmos:-18}" -# -# __layout_x__="${layout_x:-16}" -# __layout_y__="${layout_y:-48}" -# __wrtcmp_write_groups__="2" -# __wrtcmp_write_tasks_per_group__="24" -# __blocksize__="${blocksize:-32}" -# -# elif [ "${grid_gen_method}" = "ESGgrid" ]; then - - __grid_gen_method__="ESGgrid" - -# Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar -# With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar - -# Longitude and latitude for center of domain - __esggrid_lon_ctr__="-153.0" - __esggrid_lat_ctr__="61.0" - -# Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution -# of the supergrid, which is HALF of this dx - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - -# Number of x and y points for your domain (halo not included); -# Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/ak/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - __esggrid_nx__="1344" # Supergrid value 2704 - __esggrid_ny__="1152" # Supergrid value 2320 - -# Rotation of the ESG grid in degrees. - __esggrid_pazi__="0.0" - -# Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now -# Within the model we actually have a 4-point halo and a 3-point halo - __esggrid_wide_halo_width__="6" - -# Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration -# 'ksplit' is the factor that determines the timestep for this process (divided - -# Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the -# top-level routine in the dynamics is called as well as the frequency with which the physics is called. -# -# Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - - __dt_atmos__="${dt_atmos:-18}" - -#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y - __layout_x__="${layout_x:-28}" - __layout_y__="${layout_y:-16}" - -#Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. -#This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic -# For Theia, must be ~40 or less -# Check setup.sh for more details - __blocksize__="${blocksize:-24}" - -#This section is all for the write component, which you need for output during model integration - if [ "$quilting" = "TRUE" ]; then -#Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - __wrtcmp_write_groups__="1" -#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value - __wrtcmp_write_tasks_per_group__="24" -#lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - __wrtcmp_output_grid__="lambert_conformal" -#These should always be set the same as compute grid - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" -#Write component grid must always be <= compute grid (without haloes) - __wrtcmp_nx__="1344" - __wrtcmp_ny__="1152" -#Lower left latlon (southwest corner) - __wrtcmp_lon_lwr_left__="-177.0" - __wrtcmp_lat_lwr_left__="42.5" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# EMC's Hawaii grid. -# -#----------------------------------------------------------------------- -# - "EMC_HI") - - __grid_gen_method__="ESGgrid" - -# Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/hi/C768_grid.tile7.nc -# With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar -# Longitude and latitude for center of domain - __esggrid_lon_ctr__="-157.0" - __esggrid_lat_ctr__="20.0" - -# Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution -# of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - -# Number of x and y points for your domain (halo not included); -# Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/hi/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - __esggrid_nx__="432" # Supergrid value 880 - __esggrid_ny__="360" # Supergrid value 736 - -# Rotation of the ESG grid in degrees. - __esggrid_pazi__="0.0" - -# Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now -# Within the model we actually have a 4-point halo and a 3-point halo - __esggrid_wide_halo_width__="6" - -# Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration -# 'ksplit' is the factor that determines the timestep for this process (divided - -# Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the -# top-level routine in the dynamics is called as well as the frequency with which the physics is called. -# -# Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - - __dt_atmos__="${dt_atmos:-18}" - -#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y - __layout_x__="${layout_x:-8}" - __layout_y__="${layout_y:-8}" -#Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. -#This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic -# For Theia, must be ~40 or less -# Check setup.sh for more details - __blocksize__="${blocksize:-27}" - -#This section is all for the write component, which you need for output during model integration - if [ "$quilting" = "TRUE" ]; then -#Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - __wrtcmp_write_groups__="1" -#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value - __wrtcmp_write_tasks_per_group__="8" -#lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - __wrtcmp_output_grid__="lambert_conformal" -#These should usually be set the same as compute grid - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" -#Write component grid should be close to the ESGgrid values unless you are doing something weird - __wrtcmp_nx__="420" - __wrtcmp_ny__="348" - -#Lower left latlon (southwest corner) - __wrtcmp_lon_lwr_left__="-162.8" - __wrtcmp_lat_lwr_left__="15.2" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# EMC's Puerto Rico grid. -# -#----------------------------------------------------------------------- -# - "EMC_PR") - - __grid_gen_method__="ESGgrid" - -# Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/pr/C768_grid.tile7.nc -# With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar -# Longitude and latitude for center of domain - __esggrid_lon_ctr__="-69.0" - __esggrid_lat_ctr__="18.0" - -# Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution -# of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - -# Number of x and y points for your domain (halo not included); -# Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/pr/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - __esggrid_nx__="576" # Supergrid value 1168 - __esggrid_ny__="432" # Supergrid value 880 - -# Rotation of the ESG grid in degrees. - __esggrid_pazi__="0.0" - -# Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now -# Within the model we actually have a 4-point halo and a 3-point halo - __esggrid_wide_halo_width__="6" - -# Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration -# 'ksplit' is the factor that determines the timestep for this process (divided - -# Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the -# top-level routine in the dynamics is called as well as the frequency with which the physics is called. -# -# Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - - __dt_atmos__="${dt_atmos:-18}" - -#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-8}" - -#Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. -#This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic -# For Theia, must be ~40 or less -# Check setup.sh for more details - __blocksize__="${blocksize:-24}" - -#This section is all for the write component, which you need for output during model integration - if [ "$quilting" = "TRUE" ]; then -#Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - __wrtcmp_write_groups__="1" -#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value - __wrtcmp_write_tasks_per_group__="24" -#lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - __wrtcmp_output_grid__="lambert_conformal" -#These should always be set the same as compute grid - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" -#Write component grid must always be <= compute grid (without haloes) - __wrtcmp_nx__="576" - __wrtcmp_ny__="432" -#Lower left latlon (southwest corner) - __wrtcmp_lon_lwr_left__="-77" - __wrtcmp_lat_lwr_left__="12" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# EMC's Guam grid. -# -#----------------------------------------------------------------------- -# - "EMC_GU") - - __grid_gen_method__="ESGgrid" - -# Values taken from pre-generated files in /scratch4/NCEPDEV/fv3-cam/save/Benjamin.Blake/regional_workflow/fix/fix_sar/guam/C768_grid.tile7.nc -# With move to Hera, those files were lost; a backup can be found here: /scratch2/BMC/det/kavulich/fix/fix_sar -# Longitude and latitude for center of domain - __esggrid_lon_ctr__="146.0" - __esggrid_lat_ctr__="15.0" - -# Projected grid spacing in meters...in the static files (e.g. "C768_grid.tile7.nc"), the "dx" is actually the resolution -# of the supergrid, which is HALF of this dx (plus or minus some grid stretch factor) - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - -# Number of x and y points for your domain (halo not included); -# Divide "supergrid" values from /scratch2/BMC/det/kavulich/fix/fix_sar/guam/C768_grid.tile7.halo4.nc by 2 and subtract 8 to eliminate halo - __esggrid_nx__="432" # Supergrid value 880 - __esggrid_ny__="360" # Supergrid value 736 - -# Rotation of the ESG grid in degrees. - __esggrid_pazi__="0.0" - -# Number of halo points for a wide grid (before trimming)...this should almost always be 6 for now -# Within the model we actually have a 4-point halo and a 3-point halo - __esggrid_wide_halo_width__="6" - -# Side note: FV3 is lagrangian and vertical coordinates are dynamically remapped during model integration -# 'ksplit' is the factor that determines the timestep for this process (divided - -# Physics timestep in seconds, actual dynamics timestep can be a subset of this. -# This is the time step for the largest atmosphere model loop. It corresponds to the frequency with which the -# top-level routine in the dynamics is called as well as the frequency with which the physics is called. -# -# Preliminary standard values: 18 for 3-km runs, 90 for 13-km runs per config_defaults.sh - - __dt_atmos__="${dt_atmos:-18}" - -#Factors for MPI decomposition. esggrid_nx must be divisible by layout_x, esggrid_ny must be divisible by layout_y - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-12}" -#Take number of points on a tile (nx/lx*ny/ly), must divide by block size to get an integer. -#This integer must be small enough to fit into a processor's cache, so it is machine-dependent magic -# For Theia, must be ~40 or less -# Check setup.sh for more details - __blocksize__="${blocksize:-27}" - -#This section is all for the write component, which you need for output during model integration - if [ "$quilting" = "TRUE" ]; then -#Write component reserves MPI tasks for writing output. The number of "groups" is usually 1, but if you have a case where group 1 is not done writing before the next write step, you need group 2, etc. - __wrtcmp_write_groups__="1" -#Number of tasks per write group. Ny must be divisible my this number. layout_y is usually a good value - __wrtcmp_write_tasks_per_group__="24" -#lambert_conformal or rotated_latlon. lambert_conformal not well tested and probably doesn't work for our purposes - __wrtcmp_output_grid__="lambert_conformal" -#These should always be set the same as compute grid - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" -#Write component grid must always be <= compute grid (without haloes) - __wrtcmp_nx__="420" - __wrtcmp_ny__="348" -#Lower left latlon (southwest corner) Used /scratch2/NCEPDEV/fv3-cam/Dusan.Jovic/dbrowse/fv3grid utility to find best value - __wrtcmp_lon_lwr_left__="140" - __wrtcmp_lat_lwr_left__="10" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# Emulation of the HAFS v0.A grid at 25 km. -# -#----------------------------------------------------------------------- -# - "GSL_HAFSV0.A_25km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-62.0" - __esggrid_lat_ctr__="22.0" - - __esggrid_delx__="25000.0" - __esggrid_dely__="25000.0" - - __esggrid_nx__="345" - __esggrid_ny__="230" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-300}" - - __layout_x__="${layout_x:-5}" - __layout_y__="${layout_y:-5}" - __blocksize__="${blocksize:-6}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="32" - __wrtcmp_output_grid__="regional_latlon" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="25.0" - __wrtcmp_lon_lwr_left__="-114.5" - __wrtcmp_lat_lwr_left__="-5.0" - __wrtcmp_lon_upr_rght__="-9.5" - __wrtcmp_lat_upr_rght__="55.0" - __wrtcmp_dlon__="0.25" - __wrtcmp_dlat__="0.25" - fi - ;; -# -#----------------------------------------------------------------------- -# -# Emulation of the HAFS v0.A grid at 13 km. -# -#----------------------------------------------------------------------- -# - "GSL_HAFSV0.A_13km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-62.0" - __esggrid_lat_ctr__="22.0" - - __esggrid_delx__="13000.0" - __esggrid_dely__="13000.0" - - __esggrid_nx__="665" - __esggrid_ny__="444" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-180}" - - __layout_x__="${layout_x:-19}" - __layout_y__="${layout_y:-12}" - __blocksize__="${blocksize:-35}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="32" - __wrtcmp_output_grid__="regional_latlon" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="25.0" - __wrtcmp_lon_lwr_left__="-114.5" - __wrtcmp_lat_lwr_left__="-5.0" - __wrtcmp_lon_upr_rght__="-9.5" - __wrtcmp_lat_upr_rght__="55.0" - __wrtcmp_dlon__="0.13" - __wrtcmp_dlat__="0.13" - fi - ;; -# -#----------------------------------------------------------------------- -# -# Emulation of the HAFS v0.A grid at 3 km. -# -#----------------------------------------------------------------------- -# - "GSL_HAFSV0.A_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-62.0" - __esggrid_lat_ctr__="22.0" - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="2880" - __esggrid_ny__="1920" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-40}" - - __layout_x__="${layout_x:-32}" - __layout_y__="${layout_y:-24}" - __blocksize__="${blocksize:-32}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="32" - __wrtcmp_output_grid__="regional_latlon" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="25.0" - __wrtcmp_lon_lwr_left__="-114.5" - __wrtcmp_lat_lwr_left__="-5.0" - __wrtcmp_lon_upr_rght__="-9.5" - __wrtcmp_lat_upr_rght__="55.0" - __wrtcmp_dlon__="0.03" - __wrtcmp_dlat__="0.03" - fi - ;; -# -#----------------------------------------------------------------------- -# -# 50-km HRRR Alaska grid. -# -#----------------------------------------------------------------------- -# - "GSD_HRRR_AK_50km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-163.5" - __esggrid_lat_ctr__="62.8" - - __esggrid_delx__="50000.0" - __esggrid_dely__="50000.0" - - __esggrid_nx__="74" - __esggrid_ny__="51" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-600}" - - __layout_x__="${layout_x:-2}" - __layout_y__="${layout_y:-3}" - __blocksize__="${blocksize:-37}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="1" - __wrtcmp_output_grid__="lambert_conformal" - __wrtcmp_cen_lon__="${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat1__="${__esggrid_lat_ctr__}" - __wrtcmp_stdlat2__="${__esggrid_lat_ctr__}" - __wrtcmp_nx__="70" - __wrtcmp_ny__="45" - __wrtcmp_lon_lwr_left__="172.0" - __wrtcmp_lat_lwr_left__="49.0" - __wrtcmp_dx__="${__esggrid_delx__}" - __wrtcmp_dy__="${__esggrid_dely__}" - fi - ;; -# -#----------------------------------------------------------------------- -# -# Emulation of GSD's RAP domain with ~13km cell size. -# -#----------------------------------------------------------------------- -# - "RRFS_NA_13km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__="-112.5" - __esggrid_lat_ctr__="55.0" - - __esggrid_delx__="13000.0" - __esggrid_dely__="13000.0" - - __esggrid_nx__="912" - __esggrid_ny__="623" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-50}" - - __layout_x__="${layout_x:-16}" - __layout_y__="${layout_y:-16}" - __blocksize__="${blocksize:-30}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="16" - __wrtcmp_output_grid__="rotated_latlon" - __wrtcmp_cen_lon__="-113.0" #"${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="55.0" #"${__esggrid_lat_ctr__}" - __wrtcmp_lon_lwr_left__="-61.0" - __wrtcmp_lat_lwr_left__="-37.0" - __wrtcmp_lon_upr_rght__="61.0" - __wrtcmp_lat_upr_rght__="37.0" - __wrtcmp_dlon__=$( printf "%.9f" $( bc -l <<< "(${__esggrid_delx__}/${radius_Earth})*${degs_per_radian}" ) ) - __wrtcmp_dlat__=$( printf "%.9f" $( bc -l <<< "(${__esggrid_dely__}/${radius_Earth})*${degs_per_radian}" ) ) - fi - ;; -# -#----------------------------------------------------------------------- -# -# Future operational RRFS domain with ~3km cell size. -# -#----------------------------------------------------------------------- -# - "RRFS_NA_3km") - - __grid_gen_method__="ESGgrid" - - __esggrid_lon_ctr__=-112.5 - __esggrid_lat_ctr__=55.0 - - __esggrid_delx__="3000.0" - __esggrid_dely__="3000.0" - - __esggrid_nx__="3950" - __esggrid_ny__="2700" - - __esggrid_pazi__="0.0" - - __esggrid_wide_halo_width__="6" - - __dt_atmos__="${dt_atmos:-36}" - - __layout_x__="${layout_x:-20}" # 40 - EMC operational configuration - __layout_y__="${layout_y:-35}" # 45 - EMC operational configuration - __blocksize__="${blocksize:-28}" - - if [ "$quilting" = "TRUE" ]; then - __wrtcmp_write_groups__="1" - __wrtcmp_write_tasks_per_group__="144" - __wrtcmp_output_grid__="rotated_latlon" - __wrtcmp_cen_lon__="-113.0" #"${__esggrid_lon_ctr__}" - __wrtcmp_cen_lat__="55.0" #"${__esggrid_lat_ctr__}" - __wrtcmp_lon_lwr_left__="-61.0" - __wrtcmp_lat_lwr_left__="-37.0" - __wrtcmp_lon_upr_rght__="61.0" - __wrtcmp_lat_upr_rght__="37.0" - __wrtcmp_dlon__="0.025" #$( printf "%.9f" $( bc -l <<< "(${__esggrid_delx__}/${radius_Earth})*${degs_per_radian}" ) ) - __wrtcmp_dlat__="0.025" #$( printf "%.9f" $( bc -l <<< "(${__esggrid_dely__}/${radius_Earth})*${degs_per_radian}" ) ) - fi - ;; - - esac -# -#----------------------------------------------------------------------- -# -# Use the printf utility with the -v flag to set this function's output -# variables. Note that each of these is set only if the corresponding -# input variable specifying the name to use for the output variable is -# not empty. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_grid_gen_method}" ]; then - printf -v ${outvarname_grid_gen_method} "%s" "${__grid_gen_method__}" - fi - - if [ ! -z "${outvarname_esggrid_lon_ctr}" ]; then - printf -v ${outvarname_esggrid_lon_ctr} "%s" "${__esggrid_lon_ctr__}" - fi - - if [ ! -z "${outvarname_esggrid_lat_ctr}" ]; then - printf -v ${outvarname_esggrid_lat_ctr} "%s" "${__esggrid_lat_ctr__}" - fi - - if [ ! -z "${outvarname_esggrid_delx}" ]; then - printf -v ${outvarname_esggrid_delx} "%s" "${__esggrid_delx__}" - fi - - if [ ! -z "${outvarname_esggrid_dely}" ]; then - printf -v ${outvarname_esggrid_dely} "%s" "${__esggrid_dely__}" - fi - - if [ ! -z "${outvarname_esggrid_nx}" ]; then - printf -v ${outvarname_esggrid_nx} "%s" "${__esggrid_nx__}" - fi - - if [ ! -z "${outvarname_esggrid_ny}" ]; then - printf -v ${outvarname_esggrid_ny} "%s" "${__esggrid_ny__}" - fi - - if [ ! -z "${outvarname_esggrid_pazi}" ]; then - printf -v ${outvarname_esggrid_pazi} "%s" "${__esggrid_pazi__}" - fi - - if [ ! -z "${outvarname_esggrid_wide_halo_width}" ]; then - printf -v ${outvarname_esggrid_wide_halo_width} "%s" "${__esggrid_wide_halo_width__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_lon_t6_ctr}" ]; then - printf -v ${outvarname_gfdlgrid_lon_t6_ctr} "%s" "${__gfdlgrid_lon_t6_ctr__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_lat_t6_ctr}" ]; then - printf -v ${outvarname_gfdlgrid_lat_t6_ctr} "%s" "${__gfdlgrid_lat_t6_ctr__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_stretch_fac}" ]; then - printf -v ${outvarname_gfdlgrid_stretch_fac} "%s" "${__gfdlgrid_stretch_fac__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_num_cells}" ]; then - printf -v ${outvarname_gfdlgrid_num_cells} "%s" "${__gfdlgrid_num_cells__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_refine_ratio}" ]; then - printf -v ${outvarname_gfdlgrid_refine_ratio} "%s" "${__gfdlgrid_refine_ratio__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g}" ]; then - printf -v ${outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_istart_of_rgnl_dom_on_t6g__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g}" ]; then - printf -v ${outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_iend_of_rgnl_dom_on_t6g__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g}" ]; then - printf -v ${outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_jstart_of_rgnl_dom_on_t6g__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g}" ]; then - printf -v ${outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g} "%s" "${__gfdlgrid_jend_of_rgnl_dom_on_t6g__}" - fi - - if [ ! -z "${outvarname_gfdlgrid_use_num_cells_in_filenames}" ]; then - printf -v ${outvarname_gfdlgrid_use_num_cells_in_filenames} "%s" "${__gfdlgrid_use_num_cells_in_filenames__}" - fi - - if [ ! -z "${outvarname_dt_atmos}" ]; then - printf -v ${outvarname_dt_atmos} "%s" "${__dt_atmos__}" - fi - - if [ ! -z "${outvarname_layout_x}" ]; then - printf -v ${outvarname_layout_x} "%s" "${__layout_x__}" - fi - - if [ ! -z "${outvarname_layout_y}" ]; then - printf -v ${outvarname_layout_y} "%s" "${__layout_y__}" - fi - - if [ ! -z "${outvarname_blocksize}" ]; then - printf -v ${outvarname_blocksize} "%s" "${__blocksize__}" - fi - - if [ ! -z "${outvarname_wrtcmp_write_groups}" ]; then - printf -v ${outvarname_wrtcmp_write_groups} "%s" "${__wrtcmp_write_groups__}" - fi - - if [ ! -z "${outvarname_wrtcmp_write_tasks_per_group}" ]; then - printf -v ${outvarname_wrtcmp_write_tasks_per_group} "%s" "${__wrtcmp_write_tasks_per_group__}" - fi - - if [ ! -z "${outvarname_wrtcmp_output_grid}" ]; then - printf -v ${outvarname_wrtcmp_output_grid} "%s" "${__wrtcmp_output_grid__}" - fi - - if [ ! -z "${outvarname_wrtcmp_cen_lon}" ]; then - printf -v ${outvarname_wrtcmp_cen_lon} "%s" "${__wrtcmp_cen_lon__}" - fi - - if [ ! -z "${outvarname_wrtcmp_cen_lat}" ]; then - printf -v ${outvarname_wrtcmp_cen_lat} "%s" "${__wrtcmp_cen_lat__}" - fi - - if [ ! -z "${outvarname_wrtcmp_stdlat1}" ]; then - printf -v ${outvarname_wrtcmp_stdlat1} "%s" "${__wrtcmp_stdlat1__}" - fi - - if [ ! -z "${outvarname_wrtcmp_stdlat2}" ]; then - printf -v ${outvarname_wrtcmp_stdlat2} "%s" "${__wrtcmp_stdlat2__}" - fi - - if [ ! -z "${outvarname_wrtcmp_nx}" ]; then - printf -v ${outvarname_wrtcmp_nx} "%s" "${__wrtcmp_nx__}" - fi - - if [ ! -z "${outvarname_wrtcmp_ny}" ]; then - printf -v ${outvarname_wrtcmp_ny} "%s" "${__wrtcmp_ny__}" - fi - - if [ ! -z "${outvarname_wrtcmp_lon_lwr_left}" ]; then - printf -v ${outvarname_wrtcmp_lon_lwr_left} "%s" "${__wrtcmp_lon_lwr_left__}" - fi - - if [ ! -z "${outvarname_wrtcmp_lat_lwr_left}" ]; then - printf -v ${outvarname_wrtcmp_lat_lwr_left} "%s" "${__wrtcmp_lat_lwr_left__}" - fi - - if [ ! -z "${outvarname_wrtcmp_lon_upr_rght}" ]; then - printf -v ${outvarname_wrtcmp_lon_upr_rght} "%s" "${__wrtcmp_lon_upr_rght__}" - fi - - if [ ! -z "${outvarname_wrtcmp_lat_upr_rght}" ]; then - printf -v ${outvarname_wrtcmp_lat_upr_rght} "%s" "${__wrtcmp_lat_upr_rght__}" - fi - - if [ ! -z "${outvarname_wrtcmp_dx}" ]; then - printf -v ${outvarname_wrtcmp_dx} "%s" "${__wrtcmp_dx__}" - fi - - if [ ! -z "${outvarname_wrtcmp_dy}" ]; then - printf -v ${outvarname_wrtcmp_dy} "%s" "${__wrtcmp_dy__}" - fi - - if [ ! -z "${outvarname_wrtcmp_dlon}" ]; then - printf -v ${outvarname_wrtcmp_dlon} "%s" "${__wrtcmp_dlon__}" - fi - - if [ ! -z "${outvarname_wrtcmp_dlat}" ]; then - printf -v ${outvarname_wrtcmp_dlat} "%s" "${__wrtcmp_dlat__}" - fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh deleted file mode 100644 index 7bad26c2bb..0000000000 --- a/ush/set_thompson_mp_fix_files.sh +++ /dev/null @@ -1,192 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that first checks whether the Thompson -# microphysics parameterization is being called by the selected physics -# suite. If not, it sets the output variable whose name is specified by -# output_varname_sdf_uses_thompson_mp to "FALSE" and exits. If so, it -# sets this variable to "TRUE" and modifies the workflow arrays -# FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING -# to ensure that fixed files needed by the Thompson microphysics -# parameterization are copied to the FIXam directory and that appropriate -# symlinks to these files are created in the run directories. -# -#----------------------------------------------------------------------- -# -function set_thompson_mp_fix_files() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "ccpp_phys_suite_fp" \ - "thompson_mp_climo_fn" \ - "output_varname_sdf_uses_thompson_mp" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local thompson_mp_name \ - regex_search \ - thompson_mp_name_or_null \ - sdf_uses_thompson_mp \ - thompson_mp_fix_files \ - num_files \ - mapping \ - msg -# -#----------------------------------------------------------------------- -# -# Check the suite definition file to see whether the Thompson microphysics -# parameterization is being used. -# -#----------------------------------------------------------------------- -# - thompson_mp_name="mp_thompson" - regex_search="^[ ]*(${thompson_mp_name})<\/scheme>[ ]*$" - thompson_mp_name_or_null=$( $SED -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) - - if [ "${thompson_mp_name_or_null}" = "${thompson_mp_name}" ]; then - sdf_uses_thompson_mp="TRUE" - elif [ -z "${thompson_mp_name_or_null}" ]; then - sdf_uses_thompson_mp="FALSE" - else - print_err_msg_exit "\ -Unexpected value returned for thompson_mp_name_or_null: - thompson_mp_name_or_null = \"${thompson_mp_name_or_null}\" -This variable should be set to either \"${thompson_mp_name}\" or an empty -string." - fi -# -#----------------------------------------------------------------------- -# -# If the Thompson microphysics parameterization is being used, then... -# -#----------------------------------------------------------------------- -# - if [ "${sdf_uses_thompson_mp}" = "TRUE" ]; then -# -#----------------------------------------------------------------------- -# -# Append the names of the fixed files needed by the Thompson microphysics -# parameterization to the workflow array FIXgsm_FILES_TO_COPY_TO_FIXam, -# and append to the workflow array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING -# the mappings between these files and the names of the corresponding -# symlinks that need to be created in the run directories. -# -#----------------------------------------------------------------------- -# - thompson_mp_fix_files=( \ - "CCN_ACTIVATE.BIN" \ - "freezeH2O.dat" \ - "qr_acr_qg.dat" \ - "qr_acr_qs.dat" \ - "qr_acr_qgV2.dat" \ - "qr_acr_qsV2.dat" \ - ) - - if [ "${EXTRN_MDL_NAME_ICS}" != "HRRR" -a "${EXTRN_MDL_NAME_ICS}" != "RAP" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" != "HRRR" -a "${EXTRN_MDL_NAME_LBCS}" != "RAP" ]; then - thompson_mp_fix_files+=( "${thompson_mp_climo_fn}" ) - fi - - FIXgsm_FILES_TO_COPY_TO_FIXam+=( "${thompson_mp_fix_files[@]}" ) - - num_files=${#thompson_mp_fix_files[@]} - for (( i=0; i<${num_files}; i++ )); do - mapping="${thompson_mp_fix_files[i]} | ${thompson_mp_fix_files[i]}" - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING+=( "${mapping}" ) - done - - msg=" -Since the Thompson microphysics parameterization is being used by this -physics suite (CCPP_PHYS_SUITE), the names of the fixed files needed by -this scheme have been appended to the array FIXgsm_FILES_TO_COPY_TO_FIXam, -and the mappings between these files and the symlinks that need to be -created in the cycle directories have been appended to the array -CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the -values of these parameters are as follows: - - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" - - FIXgsm_FILES_TO_COPY_TO_FIXam = ( \\ -" - msg="$msg"$( printf "\"%s\" \\\\\n" "${FIXgsm_FILES_TO_COPY_TO_FIXam[@]}" ) - msg="$msg"$( printf "\n)" ) - msg="$msg - - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = ( \\ -" - msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) - msg="$msg"$( printf "\n)" ) - print_info_msg "$msg" - - fi -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - eval ${output_varname_sdf_uses_thompson_mp}="${sdf_uses_thompson_mp}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 - -} - diff --git a/ush/setup.py b/ush/setup.py index 06bf3dd535..7a3cb032f0 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -7,7 +7,6 @@ from python_utils import cd_vrfy, mkdir_vrfy, rm_vrfy, check_var_valid_value,\ lowercase,uppercase,check_for_preexist_dir_file,\ - list_to_str, type_to_str, \ import_vars, export_vars, get_env_var, print_info_msg,\ print_err_msg_exit, load_config_file, cfg_to_shell_str,\ load_shell_config, load_ini_config, get_ini_value @@ -58,8 +57,8 @@ def setup(): # #----------------------------------------------------------------------- # - EXPT_DEFAULT_CONFIG_FN="config_defaults.yaml" - cfg_d = load_config_file(EXPT_DEFAULT_CONFIG_FN) + EXPT_DEFAULT_CONFIG_FN="config_defaults.sh" + cfg_d = load_config_file(os.path.join(ushdir,EXPT_DEFAULT_CONFIG_FN)) import_vars(dictionary=cfg_d) # #----------------------------------------------------------------------- @@ -91,6 +90,14 @@ def setup(): not defined in the default configuration file {EXPT_DEFAULT_CONFIG_FN}''') import_vars(dictionary=cfg_u) + # + #----------------------------------------------------------------------- + # Source constants.sh and save its contents to a variable for later + #----------------------------------------------------------------------- + # + cfg_c=load_config_file(os.path.join(ushdir,CONSTANTS_FN)) + const_lines=cfg_to_shell_str(cfg_c) + import_vars(dictionary=cfg_c) # #----------------------------------------------------------------------- # @@ -99,13 +106,12 @@ def setup(): # #----------------------------------------------------------------------- # - # export env vars before calling another module export_vars() if PREDEF_GRID_NAME: set_predef_grid_params() - + import_vars() # @@ -119,7 +125,7 @@ def setup(): if DEBUG and not VERBOSE: print_info_msg(''' Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"...''') - VERBOSE=False + VERBOSE=True # #----------------------------------------------------------------------- @@ -924,17 +930,12 @@ def setup(): NEMS_CONFIG_FN = "nems.configure" #---------------------------------- - if DATA_TABLE_TMPL_FN is None: - DATA_TABLE_TMPL_FN = DATA_TABLE_FN - if DIAG_TABLE_TMPL_FN is None: - DIAG_TABLE_TMPL_FN = f"{DIAG_TABLE_FN}{dot_ccpp_phys_suite_or_null}" - if FIELD_TABLE_TMPL_FN is None: - FIELD_TABLE_TMPL_FN = f"{FIELD_TABLE_FN}{dot_ccpp_phys_suite_or_null}" - if MODEL_CONFIG_TMPL_FN is None: - MODEL_CONFIG_TMPL_FN = MODEL_CONFIG_FN - if NEMS_CONFIG_TMPL_FN is None: - NEMS_CONFIG_TMPL_FN = NEMS_CONFIG_FN - + DATA_TABLE_TMPL_FN = DATA_TABLE_TMPL_FN or DATA_TABLE_FN + DIAG_TABLE_TMPL_FN = f"{DIAG_TABLE_TMPL_FN or DIAG_TABLE_FN}{dot_ccpp_phys_suite_or_null}" + FIELD_TABLE_TMPL_FN = f"{FIELD_TABLE_TMPL_FN or FIELD_TABLE_FN}{dot_ccpp_phys_suite_or_null}" + MODEL_CONFIG_TMPL_FN = MODEL_CONFIG_TMPL_FN or MODEL_CONFIG_FN + NEMS_CONFIG_TMPL_FN = NEMS_CONFIG_TMPL_FN or NEMS_CONFIG_FN + DATA_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,DATA_TABLE_TMPL_FN) DIAG_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,DIAG_TABLE_TMPL_FN) FIELD_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR,FIELD_TABLE_TMPL_FN) @@ -1414,55 +1415,11 @@ def setup(): set_extrn_mdl_params() - IMPORTS = ["EXTRN_MDL_SYSBASEDIR_ICS", "EXTRN_MDL_SYSBASEDIR_LBCS", "EXTRN_MDL_LBCS_OFFSET_HRS"] + IMPORTS = ["EXTRN_MDL_LBCS_OFFSET_HRS"] import_vars(env_vars=IMPORTS) # #----------------------------------------------------------------------- # - # Any regional model must be supplied lateral boundary conditions (in - # addition to initial conditions) to be able to perform a forecast. In - # the FV3-LAM model, these boundary conditions (BCs) are supplied using a - # "halo" of grid cells around the regional domain that extend beyond the - # boundary of the domain. The model is formulated such that along with - # files containing these BCs, it needs as input the following files (in - # NetCDF format): - # - # 1) A grid file that includes a halo of 3 cells beyond the boundary of - # the domain. - # 2) A grid file that includes a halo of 4 cells beyond the boundary of - # the domain. - # 3) A (filtered) orography file without a halo, i.e. a halo of width - # 0 cells. - # 4) A (filtered) orography file that includes a halo of 4 cells beyond - # the boundary of the domain. - # - # Note that the regional grid is referred to as "tile 7" in the code. - # We will let: - # - # * NH0 denote the width (in units of number of cells on tile 7) of - # the 0-cell-wide halo, i.e. NH0 = 0; - # - # * NH3 denote the width (in units of number of cells on tile 7) of - # the 3-cell-wide halo, i.e. NH3 = 3; and - # - # * NH4 denote the width (in units of number of cells on tile 7) of - # the 4-cell-wide halo, i.e. NH4 = 4. - # - # We define these variables next. - # - #----------------------------------------------------------------------- - # - global NH0,NH3,NH4 - NH0=0 - NH3=3 - NH4=4 - - # export env vars - EXPORTS = ["NH0","NH3","NH4"] - export_vars(env_vars = EXPORTS) - # - #----------------------------------------------------------------------- - # # Set parameters according to the type of horizontal grid generation # method specified. First consider GFDL's global-parent-grid based # method. @@ -1490,7 +1447,7 @@ def setup(): set_gridparams_GFDLgrid( \ lon_of_t6_ctr=GFDLgrid_LON_T6_CTR, \ lat_of_t6_ctr=GFDLgrid_LAT_T6_CTR, \ - res_of_t6g=GFDLgrid_RES, \ + res_of_t6g=GFDLgrid_NUM_CELLS, \ stretch_factor=GFDLgrid_STRETCH_FAC, \ refine_ratio_t6g_to_t7g=GFDLgrid_REFINE_RATIO, \ istart_of_t7_on_t6g=GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G, \ @@ -1779,11 +1736,26 @@ def setup(): global GLOBAL_VAR_DEFNS_FP GLOBAL_VAR_DEFNS_FP=os.path.join(EXPTDIR,GLOBAL_VAR_DEFNS_FN) all_lines=cfg_to_shell_str(cfg_d) + with open(GLOBAL_VAR_DEFNS_FP,'w') as f: msg = f""" # + # #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 1: + # This section contains definitions of the various constants defined in + # the file {CONSTANTS_FN}. + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # + """ + f.write(dedent(msg)) + f.write(const_lines) + + msg = f""" # + #----------------------------------------------------------------------- + #----------------------------------------------------------------------- + # Section 2: # This section contains (most of) the primary experiment variables, i.e. # those variables that are defined in the default configuration file # (config_defaults.sh) and that can be reset via the user-specified @@ -1839,7 +1811,11 @@ def setup(): #----------------------------------------------------------------------- #----------------------------------------------------------------------- # + """ + with open(GLOBAL_VAR_DEFNS_FP,'a') as f: + f.write(dedent(msg)) + settings = { # #----------------------------------------------------------------------- # @@ -1849,9 +1825,9 @@ def setup(): # #----------------------------------------------------------------------- # - WFLOW_LAUNCH_SCRIPT_FP='{WFLOW_LAUNCH_SCRIPT_FP}' - WFLOW_LAUNCH_LOG_FP='{WFLOW_LAUNCH_LOG_FP}' - CRONTAB_LINE='{CRONTAB_LINE}' + 'WFLOW_LAUNCH_SCRIPT_FP': WFLOW_LAUNCH_SCRIPT_FP, + 'WFLOW_LAUNCH_LOG_FP': WFLOW_LAUNCH_LOG_FP, + 'CRONTAB_LINE': CRONTAB_LINE, # #----------------------------------------------------------------------- # @@ -1859,44 +1835,44 @@ def setup(): # #----------------------------------------------------------------------- # - SR_WX_APP_TOP_DIR='{SR_WX_APP_TOP_DIR}' - HOMErrfs='{HOMErrfs}' - USHDIR='{USHDIR}' - SCRIPTSDIR='{SCRIPTSDIR}' - JOBSDIR='{JOBSDIR}' - SORCDIR='{SORCDIR}' - SRC_DIR='{SRC_DIR}' - PARMDIR='{PARMDIR}' - MODULES_DIR='{MODULES_DIR}' - EXECDIR='{EXECDIR}' - FIXam='{FIXam}' - FIXclim='{FIXclim}' - FIXLAM='{FIXLAM}' - FIXgsm='{FIXgsm}' - FIXaer='{FIXaer}' - FIXlut='{FIXlut}' - COMROOT='{COMROOT}' - COMOUT_BASEDIR='{COMOUT_BASEDIR}' - TEMPLATE_DIR='{TEMPLATE_DIR}' - VX_CONFIG_DIR='{VX_CONFIG_DIR}' - METPLUS_CONF='{METPLUS_CONF}' - MET_CONFIG='{MET_CONFIG}' - UFS_WTHR_MDL_DIR='{UFS_WTHR_MDL_DIR}' - UFS_UTILS_DIR='{UFS_UTILS_DIR}' - SFC_CLIMO_INPUT_DIR='{SFC_CLIMO_INPUT_DIR}' - TOPO_DIR='{TOPO_DIR}' - UPP_DIR='{UPP_DIR}' + 'SR_WX_APP_TOP_DIR': SR_WX_APP_TOP_DIR, + 'HOMErrfs': HOMErrfs, + 'USHDIR': USHDIR, + 'SCRIPTSDIR': SCRIPTSDIR, + 'JOBSDIR': JOBSDIR, + 'SORCDIR': SORCDIR, + 'SRC_DIR': SRC_DIR, + 'PARMDIR': PARMDIR, + 'MODULES_DIR': MODULES_DIR, + 'EXECDIR': EXECDIR, + 'FIXam': FIXam, + 'FIXclim': FIXclim, + 'FIXLAM': FIXLAM, + 'FIXgsm': FIXgsm, + 'FIXaer': FIXaer, + 'FIXlut': FIXlut, + 'COMROOT': COMROOT, + 'COMOUT_BASEDIR': COMOUT_BASEDIR, + 'TEMPLATE_DIR': TEMPLATE_DIR, + 'VX_CONFIG_DIR': VX_CONFIG_DIR, + 'METPLUS_CONF': METPLUS_CONF, + 'MET_CONFIG': MET_CONFIG, + 'UFS_WTHR_MDL_DIR': UFS_WTHR_MDL_DIR, + 'UFS_UTILS_DIR': UFS_UTILS_DIR, + 'SFC_CLIMO_INPUT_DIR': SFC_CLIMO_INPUT_DIR, + 'TOPO_DIR': TOPO_DIR, + 'UPP_DIR': UPP_DIR, - EXPTDIR='{EXPTDIR}' - LOGDIR='{LOGDIR}' - CYCLE_BASEDIR='{CYCLE_BASEDIR}' - GRID_DIR='{GRID_DIR}' - OROG_DIR='{OROG_DIR}' - SFC_CLIMO_DIR='{SFC_CLIMO_DIR}' + 'EXPTDIR': EXPTDIR, + 'LOGDIR': LOGDIR, + 'CYCLE_BASEDIR': CYCLE_BASEDIR, + 'GRID_DIR': GRID_DIR, + 'OROG_DIR': OROG_DIR, + 'SFC_CLIMO_DIR': SFC_CLIMO_DIR, - NDIGITS_ENSMEM_NAMES='{NDIGITS_ENSMEM_NAMES}' - ENSMEM_NAMES={list_to_str(ENSMEM_NAMES)} - FV3_NML_ENSMEM_FPS={list_to_str(FV3_NML_ENSMEM_FPS)} + 'NDIGITS_ENSMEM_NAMES': NDIGITS_ENSMEM_NAMES, + 'ENSMEM_NAMES': ENSMEM_NAMES, + 'FV3_NML_ENSMEM_FPS': FV3_NML_ENSMEM_FPS, # #----------------------------------------------------------------------- # @@ -1904,49 +1880,49 @@ def setup(): # #----------------------------------------------------------------------- # - GLOBAL_VAR_DEFNS_FP='{GLOBAL_VAR_DEFNS_FP}' + 'GLOBAL_VAR_DEFNS_FP': GLOBAL_VAR_DEFNS_FP, - DATA_TABLE_FN='{DATA_TABLE_FN}' - DIAG_TABLE_FN='{DIAG_TABLE_FN}' - FIELD_TABLE_FN='{FIELD_TABLE_FN}' - MODEL_CONFIG_FN='{MODEL_CONFIG_FN}' - NEMS_CONFIG_FN='{NEMS_CONFIG_FN}' - - DATA_TABLE_TMPL_FN='{DATA_TABLE_TMPL_FN}' - DIAG_TABLE_TMPL_FN='{DIAG_TABLE_TMPL_FN}' - FIELD_TABLE_TMPL_FN='{FIELD_TABLE_TMPL_FN}' - MODEL_CONFIG_TMPL_FN='{MODEL_CONFIG_TMPL_FN}' - NEMS_CONFIG_TMPL_FN='{NEMS_CONFIG_TMPL_FN}' + 'DATA_TABLE_FN': DATA_TABLE_FN, + 'DIAG_TABLE_FN': DIAG_TABLE_FN, + 'FIELD_TABLE_FN': FIELD_TABLE_FN, + 'MODEL_CONFIG_FN': MODEL_CONFIG_FN, + 'NEMS_CONFIG_FN': NEMS_CONFIG_FN, + + 'DATA_TABLE_TMPL_FN': DATA_TABLE_TMPL_FN, + 'DIAG_TABLE_TMPL_FN': DIAG_TABLE_TMPL_FN, + 'FIELD_TABLE_TMPL_FN': FIELD_TABLE_TMPL_FN, + 'MODEL_CONFIG_TMPL_FN': MODEL_CONFIG_TMPL_FN, + 'NEMS_CONFIG_TMPL_FN': NEMS_CONFIG_TMPL_FN, - DATA_TABLE_TMPL_FP='{DATA_TABLE_TMPL_FP}' - DIAG_TABLE_TMPL_FP='{DIAG_TABLE_TMPL_FP}' - FIELD_TABLE_TMPL_FP='{FIELD_TABLE_TMPL_FP}' - FV3_NML_BASE_SUITE_FP='{FV3_NML_BASE_SUITE_FP}' - FV3_NML_YAML_CONFIG_FP='{FV3_NML_YAML_CONFIG_FP}' - FV3_NML_BASE_ENS_FP='{FV3_NML_BASE_ENS_FP}' - MODEL_CONFIG_TMPL_FP='{MODEL_CONFIG_TMPL_FP}' - NEMS_CONFIG_TMPL_FP='{NEMS_CONFIG_TMPL_FP}' + 'DATA_TABLE_TMPL_FP': DATA_TABLE_TMPL_FP, + 'DIAG_TABLE_TMPL_FP': DIAG_TABLE_TMPL_FP, + 'FIELD_TABLE_TMPL_FP': FIELD_TABLE_TMPL_FP, + 'FV3_NML_BASE_SUITE_FP': FV3_NML_BASE_SUITE_FP, + 'FV3_NML_YAML_CONFIG_FP': FV3_NML_YAML_CONFIG_FP, + 'FV3_NML_BASE_ENS_FP': FV3_NML_BASE_ENS_FP, + 'MODEL_CONFIG_TMPL_FP': MODEL_CONFIG_TMPL_FP, + 'NEMS_CONFIG_TMPL_FP': NEMS_CONFIG_TMPL_FP, - CCPP_PHYS_SUITE_FN='{CCPP_PHYS_SUITE_FN}' - CCPP_PHYS_SUITE_IN_CCPP_FP='{CCPP_PHYS_SUITE_IN_CCPP_FP}' - CCPP_PHYS_SUITE_FP='{CCPP_PHYS_SUITE_FP}' + 'CCPP_PHYS_SUITE_FN': CCPP_PHYS_SUITE_FN, + 'CCPP_PHYS_SUITE_IN_CCPP_FP': CCPP_PHYS_SUITE_IN_CCPP_FP, + 'CCPP_PHYS_SUITE_FP': CCPP_PHYS_SUITE_FP, - FIELD_DICT_FN='{FIELD_DICT_FN}' - FIELD_DICT_IN_UWM_FP='{FIELD_DICT_IN_UWM_FP}' - FIELD_DICT_FP='{FIELD_DICT_FP}' + 'FIELD_DICT_FN': FIELD_DICT_FN, + 'FIELD_DICT_IN_UWM_FP': FIELD_DICT_IN_UWM_FP, + 'FIELD_DICT_FP': FIELD_DICT_FP, - DATA_TABLE_FP='{DATA_TABLE_FP}' - FIELD_TABLE_FP='{FIELD_TABLE_FP}' - FV3_NML_FN='{FV3_NML_FN}' # This may not be necessary... - FV3_NML_FP='{FV3_NML_FP}' - NEMS_CONFIG_FP='{NEMS_CONFIG_FP}' + 'DATA_TABLE_FP': DATA_TABLE_FP, + 'FIELD_TABLE_FP': FIELD_TABLE_FP, + 'FV3_NML_FN': FV3_NML_FN, # This may not be necessary... + 'FV3_NML_FP': FV3_NML_FP, + 'NEMS_CONFIG_FP': NEMS_CONFIG_FP, - FV3_EXEC_FP='{FV3_EXEC_FP}' + 'FV3_EXEC_FP': FV3_EXEC_FP, - LOAD_MODULES_RUN_TASK_FP='{LOAD_MODULES_RUN_TASK_FP}' + 'LOAD_MODULES_RUN_TASK_FP': LOAD_MODULES_RUN_TASK_FP, - THOMPSON_MP_CLIMO_FN='{THOMPSON_MP_CLIMO_FN}' - THOMPSON_MP_CLIMO_FP='{THOMPSON_MP_CLIMO_FP}' + 'THOMPSON_MP_CLIMO_FN': THOMPSON_MP_CLIMO_FN, + 'THOMPSON_MP_CLIMO_FP': THOMPSON_MP_CLIMO_FP, # #----------------------------------------------------------------------- # @@ -1954,7 +1930,7 @@ def setup(): # #----------------------------------------------------------------------- # - RELATIVE_LINK_FLAG='{RELATIVE_LINK_FLAG}' + 'RELATIVE_LINK_FLAG': RELATIVE_LINK_FLAG, # #----------------------------------------------------------------------- # @@ -1963,8 +1939,8 @@ def setup(): # #----------------------------------------------------------------------- # - SDF_USES_RUC_LSM='{type_to_str(SDF_USES_RUC_LSM)}' - SDF_USES_THOMPSON_MP='{type_to_str(SDF_USES_THOMPSON_MP)}' + 'SDF_USES_RUC_LSM': SDF_USES_RUC_LSM, + 'SDF_USES_THOMPSON_MP': SDF_USES_THOMPSON_MP, # #----------------------------------------------------------------------- # @@ -1973,28 +1949,24 @@ def setup(): # #----------------------------------------------------------------------- # - GTYPE='{GTYPE}' - TILE_RGNL='{TILE_RGNL}' - NH0='{NH0}' - NH3='{NH3}' - NH4='{NH4}' + 'GTYPE': GTYPE, + 'TILE_RGNL': TILE_RGNL, - LON_CTR='{LON_CTR}' - LAT_CTR='{LAT_CTR}' - NX='{NX}' - NY='{NY}' - NHW='{NHW}' - STRETCH_FAC='{STRETCH_FAC}' + 'LON_CTR': LON_CTR, + 'LAT_CTR': LAT_CTR, + 'NX': NX, + 'NY': NY, + 'NHW': NHW, + 'STRETCH_FAC': STRETCH_FAC, - RES_IN_FIXLAM_FILENAMES='{RES_IN_FIXLAM_FILENAMES}' + 'RES_IN_FIXLAM_FILENAMES': RES_IN_FIXLAM_FILENAMES, # # If running the make_grid task, CRES will be set to a null string during # the grid generation step. It will later be set to an actual value after # the make_grid task is complete. # - CRES='{CRES}'""" - with open(GLOBAL_VAR_DEFNS_FP,'a') as f: - f.write(dedent(msg)) + 'CRES': CRES + } # #----------------------------------------------------------------------- # @@ -2004,8 +1976,6 @@ def setup(): #----------------------------------------------------------------------- # if GRID_GEN_METHOD == "GFDLgrid": - - msg=f""" # #----------------------------------------------------------------------- # @@ -2019,16 +1989,13 @@ def setup(): # #----------------------------------------------------------------------- # - ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' - IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' - JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' - JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='{JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}'""" - with open(GLOBAL_VAR_DEFNS_FP,'a') as f: - f.write(dedent(msg)) - + settings.update({ + 'ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG': ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + 'IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG': IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + 'JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG': JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG, + 'JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG': JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG + }) elif GRID_GEN_METHOD == "ESGgrid": - - msg=f""" # #----------------------------------------------------------------------- # @@ -2039,13 +2006,13 @@ def setup(): # #----------------------------------------------------------------------- # - DEL_ANGLE_X_SG='{DEL_ANGLE_X_SG}' - DEL_ANGLE_Y_SG='{DEL_ANGLE_Y_SG}' - NEG_NX_OF_DOM_WITH_WIDE_HALO='{NEG_NX_OF_DOM_WITH_WIDE_HALO}' - NEG_NY_OF_DOM_WITH_WIDE_HALO='{NEG_NY_OF_DOM_WITH_WIDE_HALO}' - PAZI='{PAZI or ''}'""" - with open(GLOBAL_VAR_DEFNS_FP,'a') as f: - f.write(dedent(msg)) + settings.update({ + 'DEL_ANGLE_X_SG': DEL_ANGLE_X_SG, + 'DEL_ANGLE_Y_SG': DEL_ANGLE_Y_SG, + 'NEG_NX_OF_DOM_WITH_WIDE_HALO': NEG_NX_OF_DOM_WITH_WIDE_HALO, + 'NEG_NY_OF_DOM_WITH_WIDE_HALO': NEG_NY_OF_DOM_WITH_WIDE_HALO, + 'PAZI': PAZI or '' + }) # #----------------------------------------------------------------------- # @@ -2054,7 +2021,7 @@ def setup(): # #----------------------------------------------------------------------- # - msg = f""" + settings.update({ # #----------------------------------------------------------------------- # @@ -2063,7 +2030,7 @@ def setup(): # #----------------------------------------------------------------------- # - CPL='{type_to_str(CPL)}' + 'CPL': CPL, # #----------------------------------------------------------------------- # @@ -2072,7 +2039,7 @@ def setup(): # #----------------------------------------------------------------------- # - OZONE_PARAM='{OZONE_PARAM}' + 'OZONE_PARAM': OZONE_PARAM, # #----------------------------------------------------------------------- # @@ -2084,7 +2051,7 @@ def setup(): # #----------------------------------------------------------------------- # - EXTRN_MDL_SYSBASEDIR_ICS='{EXTRN_MDL_SYSBASEDIR_ICS}' + 'EXTRN_MDL_SYSBASEDIR_ICS': EXTRN_MDL_SYSBASEDIR_ICS, # #----------------------------------------------------------------------- # @@ -2096,7 +2063,7 @@ def setup(): # #----------------------------------------------------------------------- # - EXTRN_MDL_SYSBASEDIR_LBCS='{EXTRN_MDL_SYSBASEDIR_LBCS}' + 'EXTRN_MDL_SYSBASEDIR_LBCS': EXTRN_MDL_SYSBASEDIR_LBCS, # #----------------------------------------------------------------------- # @@ -2105,7 +2072,7 @@ def setup(): # #----------------------------------------------------------------------- # - EXTRN_MDL_LBCS_OFFSET_HRS='{EXTRN_MDL_LBCS_OFFSET_HRS}' + 'EXTRN_MDL_LBCS_OFFSET_HRS': EXTRN_MDL_LBCS_OFFSET_HRS, # #----------------------------------------------------------------------- # @@ -2114,7 +2081,7 @@ def setup(): # #----------------------------------------------------------------------- # - LBC_SPEC_FCST_HRS={list_to_str(LBC_SPEC_FCST_HRS)} + 'LBC_SPEC_FCST_HRS': LBC_SPEC_FCST_HRS, # #----------------------------------------------------------------------- # @@ -2123,8 +2090,8 @@ def setup(): # #----------------------------------------------------------------------- # - NUM_CYCLES='{NUM_CYCLES}' - ALL_CDATES={list_to_str(ALL_CDATES)} + 'NUM_CYCLES': NUM_CYCLES, + 'ALL_CDATES': ALL_CDATES, # #----------------------------------------------------------------------- # @@ -2138,9 +2105,9 @@ def setup(): # #----------------------------------------------------------------------- # - USE_FVCOM='{type_to_str(USE_FVCOM)}' - FVCOM_DIR='{FVCOM_DIR}' - FVCOM_FILE='{FVCOM_FILE}' + 'USE_FVCOM': USE_FVCOM, + 'FVCOM_DIR': FVCOM_DIR, + 'FVCOM_FILE': FVCOM_FILE, # #----------------------------------------------------------------------- # @@ -2148,8 +2115,8 @@ def setup(): # #----------------------------------------------------------------------- # - NCORES_PER_NODE='{NCORES_PER_NODE}' - PE_MEMBER01='{PE_MEMBER01}' + 'NCORES_PER_NODE': NCORES_PER_NODE, + 'PE_MEMBER01': PE_MEMBER01, # #----------------------------------------------------------------------- # @@ -2162,17 +2129,24 @@ def setup(): # #----------------------------------------------------------------------- # - N_VAR_SPP='{N_VAR_SPP}' - N_VAR_LNDP='{N_VAR_LNDP}' - LNDP_TYPE='{LNDP_TYPE}' - LNDP_MODEL_TYPE='{LNDP_MODEL_TYPE}' - FHCYC_LSM_SPP_OR_NOT='{FHCYC_LSM_SPP_OR_NOT}' - """ + 'N_VAR_SPP': N_VAR_SPP, + 'N_VAR_LNDP': N_VAR_LNDP, + 'LNDP_TYPE': LNDP_TYPE, + 'LNDP_MODEL_TYPE': LNDP_MODEL_TYPE, + 'FHCYC_LSM_SPP_OR_NOT': FHCYC_LSM_SPP_OR_NOT + }) + # + #----------------------------------------------------------------------- + # + # Now write all settings we collacted so far to var_defns file + # + #----------------------------------------------------------------------- + # with open(GLOBAL_VAR_DEFNS_FP,'a') as f: - f.write(dedent(msg)) + f.write(cfg_to_shell_str(settings)) - # export all vars + # export all global variables back to the environment export_vars() # diff --git a/ush/setup.sh b/ush/setup.sh deleted file mode 100755 index 8ba9eab852..0000000000 --- a/ush/setup.sh +++ /dev/null @@ -1,2853 +0,0 @@ -#!/bin/bash -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets a secondary set -# of parameters needed by the various scripts that are called by the -# FV3-LAM rocoto community workflow. This secondary set of parameters is -# calculated using the primary set of user-defined parameters in the de- -# fault and custom experiment/workflow configuration scripts (whose file -# names are defined below). This script then saves both sets of parame- -# ters in a global variable definitions file (really a bash script) in -# the experiment directory. This file then gets sourced by the various -# scripts called by the tasks in the workflow. -# -#----------------------------------------------------------------------- -# -function setup() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# -cd_vrfy ${scrfunc_dir} -# -#----------------------------------------------------------------------- -# -# Source bash utility functions. -# -#----------------------------------------------------------------------- -# - -. ./source_util_funcs.sh - -print_info_msg " -======================================================================== -Starting function ${func_name}() in \"${scrfunc_fn}\"... -========================================================================" -# -#----------------------------------------------------------------------- -# -# Source other necessary files. -# -#----------------------------------------------------------------------- -# -. ./check_expt_config_vars.sh -. ./set_cycle_dates.sh -. ./set_predef_grid_params.sh -. ./set_gridparams_GFDLgrid.sh -. ./set_gridparams_ESGgrid.sh -. ./link_fix.sh -. ./set_ozone_param.sh -. ./set_thompson_mp_fix_files.sh -. ./check_ruc_lsm.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Set the name of the configuration file containing default values for -# the experiment/workflow variables. Then source the file. -# -#----------------------------------------------------------------------- -# -EXPT_DEFAULT_CONFIG_FN="config_defaults.sh" -. ./${EXPT_DEFAULT_CONFIG_FN} -# -#----------------------------------------------------------------------- -# -# If a user-specified configuration file exists, source it. This file -# contains user-specified values for a subset of the experiment/workflow -# variables that override their default values. Note that the user- -# specified configuration file is not tracked by the repository, whereas -# the default configuration file is tracked. -# -#----------------------------------------------------------------------- -# -if [ -f "${EXPT_CONFIG_FN}" ]; then -# -# We require that the variables being set in the user-specified configu- -# ration file have counterparts in the default configuration file. This -# is so that we do not introduce new variables in the user-specified -# configuration file without also officially introducing them in the de- -# fault configuration file. Thus, before sourcing the user-specified -# configuration file, we check that all variables in the user-specified -# configuration file are also assigned default values in the default -# configuration file. -# - check_expt_config_vars \ - default_config_fp="./${EXPT_DEFAULT_CONFIG_FN}" \ - config_fp="./${EXPT_CONFIG_FN}" -# -# Now source the user-specified configuration file. -# - . ./${EXPT_CONFIG_FN} -# -fi -# -#----------------------------------------------------------------------- -# -# Source the script defining the valid values of experiment variables. -# -#----------------------------------------------------------------------- -# -. ./valid_param_vals.sh -# -#----------------------------------------------------------------------- -# -# Make sure that user-defined variables are set to valid values -# -# Set binary switch variables to either "TRUE" or "FALSE" by calling -# boolify so we don't have to consider other valid values later on. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_ENVIR" "valid_vals_RUN_ENVIR" - -check_var_valid_value "VERBOSE" "valid_vals_BOOLEAN" -VERBOSE=$(boolify "$VERBOSE") - -check_var_valid_value "DEBUG" "valid_vals_BOOLEAN" -DEBUG=$(boolify "$DEBUG") - -check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_BOOLEAN" -USE_CRON_TO_RELAUNCH=$(boolify "${USE_CRON_TO_RELAUNCH}") -# -#----------------------------------------------------------------------- -# -# If DEBUG is set to "TRUE", set VERBOSE to "TRUE" to print out all -# of the VERBOSE output (in addition to any DEBUG output). -# -#----------------------------------------------------------------------- -# -if [ "$DEBUG" = "TRUE" ]; then - print_info_msg " -Setting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." - VERBOSE="TRUE" -fi -# -#----------------------------------------------------------------------- -# -# Check flags that turn on/off various workflow tasks. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_BOOLEAN" -RUN_TASK_MAKE_GRID=$(boolify "${RUN_TASK_MAKE_GRID}") - -check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_BOOLEAN" -RUN_TASK_MAKE_OROG=$(boolify "${RUN_TASK_MAKE_OROG}") - -check_var_valid_value "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_BOOLEAN" -RUN_TASK_MAKE_SFC_CLIMO=$(boolify "${RUN_TASK_MAKE_SFC_CLIMO}") - -check_var_valid_value "RUN_TASK_GET_EXTRN_ICS" "valid_vals_BOOLEAN" -RUN_TASK_GET_EXTRN_ICS=$(boolify "${RUN_TASK_GET_EXTRN_ICS}") - -check_var_valid_value "RUN_TASK_GET_EXTRN_LBCS" "valid_vals_BOOLEAN" -RUN_TASK_GET_EXTRN_LBCS=$(boolify "${RUN_TASK_GET_EXTRN_LBCS}") - -check_var_valid_value "RUN_TASK_RUN_FCST" "valid_vals_BOOLEAN" -RUN_TASK_RUN_FCST=$(boolify "${RUN_TASK_RUN_FCST}") - -check_var_valid_value "RUN_TASK_RUN_POST" "valid_vals_BOOLEAN" -RUN_TASK_RUN_POST=$(boolify "${RUN_TASK_RUN_POST}") - -check_var_valid_value "RUN_TASK_GET_OBS_CCPA" "valid_vals_BOOLEAN" -RUN_TASK_GET_OBS_CCPA=$(boolify "${RUN_TASK_GET_OBS_CCPA}") - -check_var_valid_value "RUN_TASK_GET_OBS_MRMS" "valid_vals_BOOLEAN" -RUN_TASK_GET_OBS_MRMS=$(boolify "${RUN_TASK_GET_OBS_MRMS}") - -check_var_valid_value "RUN_TASK_GET_OBS_NDAS" "valid_vals_BOOLEAN" -RUN_TASK_GET_OBS_NDAS=$(boolify "${RUN_TASK_GET_OBS_NDAS}") - -check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_BOOLEAN" -RUN_TASK_VX_GRIDSTAT=$(boolify "${RUN_TASK_VX_GRIDSTAT}") - -check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_BOOLEAN" -RUN_TASK_VX_POINTSTAT=$(boolify "${RUN_TASK_VX_POINTSTAT}") - -check_var_valid_value "RUN_TASK_VX_ENSGRID" "valid_vals_BOOLEAN" -RUN_TASK_VX_ENSGRID=$(boolify "${RUN_TASK_VX_ENSGRID}") - -check_var_valid_value "RUN_TASK_VX_ENSPOINT" "valid_vals_BOOLEAN" -RUN_TASK_VX_ENSPOINT=$(boolify "${RUN_TASK_VX_ENSPOINT}") -# -#----------------------------------------------------------------------- -# -# Check stochastic physcs flags. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_SHUM" "valid_vals_BOOLEAN" -DO_SHUM=$(boolify "${DO_SHUM}") - -check_var_valid_value "DO_SPPT" "valid_vals_BOOLEAN" -DO_SPPT=$(boolify "${DO_SPPT}") - -check_var_valid_value "DO_SKEB" "valid_vals_BOOLEAN" -DO_SKEB=$(boolify "${DO_SKEB}") - -check_var_valid_value "DO_SPP" "valid_vals_BOOLEAN" -DO_SPP=$(boolify "${DO_SPP}") - -check_var_valid_value "DO_LSM_SPP" "valid_vals_BOOLEAN" -DO_LSM_SPP=$(boolify "${DO_LSM_SPP}") - -check_var_valid_value "USE_ZMTNBLCK" "valid_vals_BOOLEAN" -USE_ZMTNBLCK=$(boolify "${USE_ZMTNBLCK}") -# -#----------------------------------------------------------------------- -# -# Set magnitude of stochastic ad-hoc schemes to -999.0 if they are not -# being used. This is required at the moment, since "do_shum/sppt/skeb" -# does not override the use of the scheme unless the magnitude is also -# specifically set to -999.0. If all "do_shum/sppt/skeb" are set to -# "false," then none will run, regardless of the magnitude values. -# -#----------------------------------------------------------------------- -# -if [ "${DO_SHUM}" = "FALSE" ]; then - SHUM_MAG=-999.0 -fi -if [ "${DO_SKEB}" = "FALSE" ]; then - SKEB_MAG=-999.0 -fi -if [ "${DO_SPPT}" = "FALSE" ]; then - SPPT_MAG=-999.0 -fi -# -#----------------------------------------------------------------------- -# -# If running with SPP in MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or -# RRTMG, count the number of entries in SPP_VAR_LIST to correctly set -# N_VAR_SPP, otherwise set it to zero. -# -#----------------------------------------------------------------------- -# -N_VAR_SPP=0 -if [ "${DO_SPP}" = "TRUE" ]; then - N_VAR_SPP=${#SPP_VAR_LIST[@]} -fi -# -#----------------------------------------------------------------------- -# -# If running with Noah or RUC-LSM SPP, count the number of entries in -# LSM_SPP_VAR_LIST to correctly set N_VAR_LNDP, otherwise set it to zero. -# Also set LNDP_TYPE to 2 for LSM SPP, otherwise set it to zero. Finally, -# initialize an "FHCYC_LSM_SPP" variable to 0 and set it to 999 if LSM SPP -# is turned on. This requirement is necessary since LSM SPP cannot run with -# FHCYC=0 at the moment, but FHCYC cannot be set to anything less than the -# length of the forecast either. A bug fix will be submitted to -# ufs-weather-model soon, at which point, this requirement can be removed -# from regional_workflow. -# -#----------------------------------------------------------------------- -# -N_VAR_LNDP=0 -LNDP_TYPE=0 -LNDP_MODEL_TYPE=0 -FHCYC_LSM_SPP_OR_NOT=0 -if [ "${DO_LSM_SPP}" = "TRUE" ]; then - N_VAR_LNDP=${#LSM_SPP_VAR_LIST[@]} - LNDP_TYPE=2 - LNDP_MODEL_TYPE=2 - FHCYC_LSM_SPP_OR_NOT=999 -fi -# -#----------------------------------------------------------------------- -# -# If running with SPP, confirm that each SPP-related namelist value -# contains the same number of entries as N_VAR_SPP (set above to be equal -# to the number of entries in SPP_VAR_LIST). -# -#----------------------------------------------------------------------- -# -if [ "${DO_SPP}" = "TRUE" ]; then - if [ "${#SPP_MAG_LIST[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#SPP_LSCALE[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#SPP_TSCALE[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#SPP_SIGTOP1[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#SPP_SIGTOP2[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#SPP_STDDEV_CUTOFF[@]}" != "${N_VAR_SPP}" ] || \ - [ "${#ISEED_SPP[@]}" != "${N_VAR_SPP}" ]; then - print_err_msg_exit "\ -All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist -variables set in ${CONFIG_FN} must be equal in number of entries to what is -found in SPP_VAR_LIST: - Number of entries in SPP_VAR_LIST = \"${#SPP_VAR_LIST[@]}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# If running with LSM SPP, confirm that each LSM SPP-related namelist -# value contains the same number of entries as N_VAR_LNDP (set above to -# be equal to the number of entries in LSM_SPP_VAR_LIST). -# -#----------------------------------------------------------------------- -# -if [ "${DO_LSM_SPP}" = "TRUE" ]; then - if [ "${#LSM_SPP_MAG_LIST[@]}" != "${N_VAR_LNDP}" ] || \ - [ "${#LSM_SPP_LSCALE[@]}" != "${N_VAR_LNDP}" ] || \ - [ "${#LSM_SPP_TSCALE[@]}" != "${N_VAR_LNDP}" ]; then - print_err_msg_exit "\ -All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) -set in ${CONFIG_FN} must be equal in number of entries to what is found in -SPP_VAR_LIST: - Number of entries in SPP_VAR_LIST = \"${#LSM_SPP_VAR_LIST[@]}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DOT_OR_USCORE is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DOT_OR_USCORE" "valid_vals_DOT_OR_USCORE" -# -#----------------------------------------------------------------------- -# -# Make sure that USE_FVCOM is set to a valid value and assign directory -# and file names. -# -# Make sure that FVCOM_WCSTART is set to lowercase "warm" or "cold" -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_FVCOM" "valid_vals_BOOLEAN" -USE_FVCOM=$(boolify "${USE_FVCOM}") - -check_var_valid_value "FVCOM_WCSTART" "valid_vals_FVCOM_WCSTART" -FVCOM_WCSTART=$(echo_lowercase $FVCOM_WCSTART) -# -#----------------------------------------------------------------------- -# -# Set various directories. -# -# HOMErrfs: -# Top directory of the clone of the FV3-LAM workflow git repository. -# -# USHDIR: -# Directory containing the shell scripts called by the workflow. -# -# SCRIPTSDIR: -# Directory containing the ex scripts called by the workflow. -# -# JOBSSDIR: -# Directory containing the jjobs scripts called by the workflow. -# -# SORCDIR: -# Directory containing various source codes. -# -# PARMDIR: -# Directory containing parameter files, template files, etc. -# -# EXECDIR: -# Directory containing various executable files. -# -# TEMPLATE_DIR: -# Directory in which templates of various FV3-LAM input files are locat- -# ed. -# -# UFS_WTHR_MDL_DIR: -# Directory in which the (NEMS-enabled) FV3-LAM application is located. -# This directory includes subdirectories for FV3, NEMS, and FMS. -# -#----------------------------------------------------------------------- -# - -# -# The current script should be located in the ush subdirectory of the -# workflow directory. Thus, the workflow directory is the one above the -# directory of the current script. -# -SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} - -# -#----------------------------------------------------------------------- -# -# Set the base directories in which codes obtained from external reposi- -# tories (using the manage_externals tool) are placed. Obtain the rela- -# tive paths to these directories by reading them in from the manage_ex- -# ternals configuration file. (Note that these are relative to the lo- -# cation of the configuration file.) Then form the full paths to these -# directories. Finally, make sure that each of these directories actu- -# ally exists. -# -#----------------------------------------------------------------------- -# -mng_extrns_cfg_fn=$( $READLINK -f "${SR_WX_APP_TOP_DIR}/Externals.cfg" ) -property_name="local_path" -# -# Get the path to the workflow scripts -# -external_name=regional_workflow -HOMErrfs=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." -HOMErrfs="${SR_WX_APP_TOP_DIR}/${HOMErrfs}" -set +x -# -# Get the base directory of the FV3 forecast model code. -# -external_name="${FCST_MODEL}" -UFS_WTHR_MDL_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." - -UFS_WTHR_MDL_DIR="${SR_WX_APP_TOP_DIR}/${UFS_WTHR_MDL_DIR}" -if [ ! -d "${UFS_WTHR_MDL_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the FV3 source code should be located -(UFS_WTHR_MDL_DIR) does not exist: - UFS_WTHR_MDL_DIR = \"${UFS_WTHR_MDL_DIR}\" -Please clone the external repository containing the code in this directory, -build the executable, and then rerun the workflow." -fi -# -# Get the base directory of the UFS_UTILS codes. -# -external_name="ufs_utils" -UFS_UTILS_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." - -UFS_UTILS_DIR="${SR_WX_APP_TOP_DIR}/${UFS_UTILS_DIR}" -if [ ! -d "${UFS_UTILS_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the UFS utilities source codes should be lo- -cated (UFS_UTILS_DIR) does not exist: - UFS_UTILS_DIR = \"${UFS_UTILS_DIR}\" -Please clone the external repository containing the code in this direct- -ory, build the executables, and then rerun the workflow." -fi -# -# Get the base directory of the UPP code. -# -external_name="UPP" -UPP_DIR=$( \ -get_manage_externals_config_property \ -"${mng_extrns_cfg_fn}" "${external_name}" "${property_name}" ) || \ -print_err_msg_exit "\ -Call to function get_manage_externals_config_property failed." - -UPP_DIR="${SR_WX_APP_TOP_DIR}/${UPP_DIR}" -if [ ! -d "${UPP_DIR}" ]; then - print_err_msg_exit "\ -The base directory in which the UPP source code should be located -(UPP_DIR) does not exist: - UPP_DIR = \"${UPP_DIR}\" -Please clone the external repository containing the code in this directory, -build the executable, and then rerun the workflow." -fi -# -# Define some other useful paths -# -USHDIR="$HOMErrfs/ush" -SCRIPTSDIR="$HOMErrfs/scripts" -JOBSDIR="$HOMErrfs/jobs" -SORCDIR="$HOMErrfs/sorc" -SRC_DIR="${SR_WX_APP_TOP_DIR}/src" -PARMDIR="$HOMErrfs/parm" -MODULES_DIR="$HOMErrfs/modulefiles" -EXECDIR="${SR_WX_APP_TOP_DIR}/${EXEC_SUBDIR}" -TEMPLATE_DIR="$USHDIR/templates" -VX_CONFIG_DIR="$TEMPLATE_DIR/parm" -METPLUS_CONF="$TEMPLATE_DIR/parm/metplus" -MET_CONFIG="$TEMPLATE_DIR/parm/met" -# -#----------------------------------------------------------------------- -# -# Convert machine name to upper case if necessary. Then make sure that -# MACHINE is set to a valid value. -# -#----------------------------------------------------------------------- -# -MACHINE=$( printf "%s" "$MACHINE" | $SED -e 's/\(.*\)/\U\1/' ) -check_var_valid_value "MACHINE" "valid_vals_MACHINE" -# -#----------------------------------------------------------------------- -# -# Source the machine config file containing architechture information, -# queue names, and supported input file paths. -# -#----------------------------------------------------------------------- -# -RELATIVE_LINK_FLAG="--relative" -MACHINE_FILE=${MACHINE_FILE:-${USHDIR}/machine/$(echo_lowercase $MACHINE).sh} -source $USHDIR/source_machine_file.sh - -if [ -z "${NCORES_PER_NODE:-}" ]; then - print_err_msg_exit "\ - NCORES_PER_NODE has not been specified in the file ${MACHINE_FILE} - Please ensure this value has been set for your desired platform. " -fi - -if [ -z "$FIXgsm" -o -z "$FIXaer" -o -z "$FIXlut" -o -z "$TOPO_DIR" -o -z "$SFC_CLIMO_INPUT_DIR" ]; then - print_err_msg_exit "\ -One or more fix file directories have not been specified for this machine: - MACHINE = \"$MACHINE\" - FIXgsm = \"${FIXgsm:-\"\"} - FIXaer = \"${FIXaer:-\"\"} - FIXlut = \"${FIXlut:-\"\"} - TOPO_DIR = \"${TOPO_DIR:-\"\"} - SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} - DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR:-\"\"} -You can specify the missing location(s) in ${machine_file}" -fi -# -#----------------------------------------------------------------------- -# -# Make sure COMPILER is set to a valid value. -# -#----------------------------------------------------------------------- -# -COMPILER=$(echo_lowercase $COMPILER) -check_var_valid_value "COMPILER" "valid_vals_COMPILER" -# -#----------------------------------------------------------------------- -# -# Set the names of the build and workflow module files (if not already -# specified by the user). These are the files that need to be loaded -# before building the component SRW App codes and running various workflow -# scripts, respectively. -# -#----------------------------------------------------------------------- -# -machine=$(echo_lowercase ${MACHINE}) -WFLOW_MOD_FN=${WFLOW_MOD_FN:-"wflow_${machine}"} -BUILD_MOD_FN=${BUILD_MOD_FN:-"build_${machine}_${COMPILER}"} -# -#----------------------------------------------------------------------- -# -# Calculate a default value for the number of processes per node for the -# RUN_FCST_TN task. Then set PPN_RUN_FCST to this default value if -# PPN_RUN_FCST is not already specified by the user. -# -#----------------------------------------------------------------------- -# -ppn_run_fcst_default="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" -PPN_RUN_FCST=${PPN_RUN_FCST:-${ppn_run_fcst_default}} -# -#----------------------------------------------------------------------- -# -# Make sure SCHED is set to a valid value. -# -#----------------------------------------------------------------------- -# -SCHED=$(echo_lowercase $SCHED) -check_var_valid_value "SCHED" "valid_vals_SCHED" -# -#----------------------------------------------------------------------- -# -# If we are using a workflow manager check that the ACCOUNT variable is -# not empty. -# -#----------------------------------------------------------------------- -# -if [ "$WORKFLOW_MANAGER" != "none" ]; then - if [ -z "$ACCOUNT" ]; then - print_err_msg_exit "\ -The variable ACCOUNT cannot be empty if you are using a workflow manager: - ACCOUNT = \"$ACCOUNT\" - WORKFLOW_MANAGER = \"$WORKFLOW_MANAGER\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Set the grid type (GTYPE). In general, in the FV3 code, this can take -# on one of the following values: "global", "stretch", "nest", and "re- -# gional". The first three values are for various configurations of a -# global grid, while the last one is for a regional grid. Since here we -# are only interested in a regional grid, GTYPE must be set to "region- -# al". -# -#----------------------------------------------------------------------- -# -GTYPE="regional" -TILE_RGNL="7" -# -#----------------------------------------------------------------------- -# -# Make sure that GTYPE is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "GTYPE" "valid_vals_GTYPE" -# -#----------------------------------------------------------------------- -# -# Make sure PREDEF_GRID_NAME is set to a valid value. -# -#----------------------------------------------------------------------- -# -if [ ! -z ${PREDEF_GRID_NAME} ]; then - err_msg="\ -The predefined regional grid specified in PREDEF_GRID_NAME is not sup- -ported: - PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\"" - check_var_valid_value \ - "PREDEF_GRID_NAME" "valid_vals_PREDEF_GRID_NAME" "${err_msg}" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that PREEXISTING_DIR_METHOD is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value \ - "PREEXISTING_DIR_METHOD" "valid_vals_PREEXISTING_DIR_METHOD" -# -#----------------------------------------------------------------------- -# -# Make sure CCPP_PHYS_SUITE is set to a valid value. -# -#----------------------------------------------------------------------- -# -err_msg="\ -The CCPP physics suite specified in CCPP_PHYS_SUITE is not supported: - CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" -check_var_valid_value \ - "CCPP_PHYS_SUITE" "valid_vals_CCPP_PHYS_SUITE" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# Make sure that USE_MERRA_CLIMO is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_MERRA_CLIMO" "valid_vals_BOOLEAN" -USE_MERRA_CLIMO=$(boolify "${USE_MERRA_CLIMO}") -# Force to "TRUE" in case of FV3_GFS_v15_thompson_mynn_lam3km: -if [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v15_thompson_mynn_lam3km" ]; then - USE_MERRA_CLIMO="TRUE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that FCST_MODEL is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "FCST_MODEL" "valid_vals_FCST_MODEL" -# -#----------------------------------------------------------------------- -# -# Set CPL to TRUE/FALSE based on FCST_MODEL. -# -#----------------------------------------------------------------------- -# -if [ "${FCST_MODEL}" = "ufs-weather-model" ]; then - CPL="FALSE" -elif [ "${FCST_MODEL}" = "fv3gfs_aqm" ]; then - CPL="TRUE" -else - print_err_msg_exit "\ -The coupling flag CPL has not been specified for this value of FCST_MODEL: - FCST_MODEL = \"${FCST_MODEL}\"" -fi -# -#----------------------------------------------------------------------- -# -# Make sure RESTART_INTERVAL is set to an integer value if present -# -#----------------------------------------------------------------------- -# -if ! [[ "${RESTART_INTERVAL}" =~ ^[0-9]+$ ]]; then - print_err_msg_exit "\ -RESTART_INTERVAL must be set to an integer number of hours. - RESTART_INTERVAL = \"${RESTART_INTERVAL}\"" -fi -# -#----------------------------------------------------------------------- -# -# Check that DATE_FIRST_CYCL and DATE_LAST_CYCL are strings consisting -# of exactly 8 digits. -# -#----------------------------------------------------------------------- -# -date_or_null=$( printf "%s" "${DATE_FIRST_CYCL}" | \ - $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${date_or_null}" ]; then - print_err_msg_exit "\ -DATE_FIRST_CYCL must be a string consisting of exactly 8 digits of the -form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit -month, and DD is the 2-digit day-of-month. - DATE_FIRST_CYCL = \"${DATE_FIRST_CYCL}\"" -fi - -date_or_null=$( printf "%s" "${DATE_LAST_CYCL}" | \ - $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${date_or_null}" ]; then - print_err_msg_exit "\ -DATE_LAST_CYCL must be a string consisting of exactly 8 digits of the -form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit -month, and DD is the 2-digit day-of-month. - DATE_LAST_CYCL = \"${DATE_LAST_CYCL}\"" -fi -# -#----------------------------------------------------------------------- -# -# Check that all elements of CYCL_HRS are strings consisting of exactly -# 2 digits that are between "00" and "23", inclusive. -# -#----------------------------------------------------------------------- -# -cycl_hrs_str=$(printf "\"%s\" " "${CYCL_HRS[@]}") -cycl_hrs_str="( ${cycl_hrs_str})" - -i=0 -for cycl_hr in "${CYCL_HRS[@]}"; do - - cycl_hr_or_null=$( printf "%s" "${cycl_hr}" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) - - if [ -z "${cycl_hr_or_null}" ]; then - print_err_msg_exit "\ -Each element of CYCL_HRS must be a string consisting of exactly 2 digits -(including a leading \"0\", if necessary) specifying an hour-of-day. -Element #$i of CYCL_HRS (where the index of the first element is 0) does -not have this form: - CYCL_HRS = ${cycl_hrs_str} - CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" - fi - - if [ "${cycl_hr_or_null}" -lt "0" ] || \ - [ "${cycl_hr_or_null}" -gt "23" ]; then - print_err_msg_exit "\ -Each element of CYCL_HRS must be an integer between \"00\" and \"23\", -inclusive (including a leading \"0\", if necessary), specifying an hour- -of-day. Element #$i of CYCL_HRS (where the index of the first element -is 0) does not have this form: - CYCL_HRS = ${cycl_hrs_str} - CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" - fi - - i=$(( $i+1 )) - -done -# -#----------------------------------------------------------------------- -# Check cycle increment for cycle frequency (cycl_freq). -# only if INCR_CYCL_FREQ < 24. -#----------------------------------------------------------------------- -# -if [ "${INCR_CYCL_FREQ}" -lt "24" ] && [ "$i" -gt "1" ]; then - cycl_intv="$(( 24/$i ))" - cycl_intv=( $( printf "%02d " "${cycl_intv}" ) ) - INCR_CYCL_FREQ=( $( printf "%02d " "${INCR_CYCL_FREQ}" ) ) - if [ "${cycl_intv}" -ne "${INCR_CYCL_FREQ}" ]; then - print_err_msg_exit "\ -The number of CYCL_HRS does not match with that expected by INCR_CYCL_FREQ: - INCR_CYCL_FREQ = ${INCR_CYCL_FREQ} - cycle interval by the number of CYCL_HRS = ${cycl_intv} - CYCL_HRS = ${cycl_hrs_str}" - fi - - im1=$(( $i-1 )) - for itmp in $( seq 1 ${im1} ); do - itm1=$(( ${itmp}-1 )) - cycl_next_itmp="$(( ${CYCL_HRS[itm1]} + ${INCR_CYCL_FREQ} ))" - cycl_next_itmp=( $( printf "%02d " "${cycl_next_itmp}" ) ) - if [ "${cycl_next_itmp}" -ne "${CYCL_HRS[$itmp]}" ]; then - print_err_msg_exit "\ -Element #${itmp} of CYCL_HRS does not match with the increment of cycle -frequency INCR_CYCL_FREQ: - CYCL_HRS = ${cycl_hrs_str} - INCR_CYCL_FREQ = ${INCR_CYCL_FREQ} - CYCL_HRS[$itmp] = \"${CYCL_HRS[$itmp]}\"" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Call a function to generate the array ALL_CDATES containing the cycle -# dates/hours for which to run forecasts. The elements of this array -# will have the form YYYYMMDDHH. They are the starting dates/times of -# the forecasts that will be run in the experiment. Then set NUM_CYCLES -# to the number of elements in this array. -# -#----------------------------------------------------------------------- -# -set_cycle_dates \ - date_start="${DATE_FIRST_CYCL}" \ - date_end="${DATE_LAST_CYCL}" \ - cycle_hrs="${cycl_hrs_str}" \ - incr_cycl_freq="${INCR_CYCL_FREQ}" \ - output_varname_all_cdates="ALL_CDATES" - -NUM_CYCLES="${#ALL_CDATES[@]}" - -if [ $NUM_CYCLES -gt 90 ] ; then - unset ALL_CDATES - print_info_msg "$VERBOSE" " -Too many cycles in ALL_CDATES to list, redefining in abbreviated form." -ALL_CDATES="${DATE_FIRST_CYCL}${CYCL_HRS[0]}...${DATE_LAST_CYCL}${CYCL_HRS[-1]}" -fi - -# -#----------------------------------------------------------------------- -# -# If using a custom post configuration file, make sure that it exists. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_CUSTOM_POST_CONFIG_FILE" "valid_vals_BOOLEAN" -USE_CUSTOM_POST_CONFIG_FILE=$(boolify "${USE_CUSTOM_POST_CONFIG_FILE}") - -if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then - if [ ! -f "${CUSTOM_POST_CONFIG_FP}" ]; then - print_err_msg_exit " -The custom post configuration specified by CUSTOM_POST_CONFIG_FP does not -exist: - CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Ensure that USE_CRTM is set to a valid value. Then, if it is set to -# "TRUE" (i.e. if using external CRTM fix files to allow post-processing -# of synthetic satellite products from the UPP, make sure that the fix -# file directory exists. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_CRTM" "valid_vals_BOOLEAN" -USE_CRTM=$(boolify "${USE_CRTM}") - -if [ ${USE_CRTM} = "TRUE" ]; then - if [ ! -d "${CRTM_DIR}" ]; then - print_err_msg_exit " -The external CRTM fix file directory specified by CRTM_DIR does not exist: - CRTM_DIR = \"${CRTM_DIR}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# The forecast length (in integer hours) cannot contain more than 3 cha- -# racters. Thus, its maximum value is 999. Check whether the specified -# forecast length exceeds this maximum value. If so, print out a warn- -# ing and exit this script. -# -#----------------------------------------------------------------------- -# -fcst_len_hrs_max="999" -if [ "${FCST_LEN_HRS}" -gt "${fcst_len_hrs_max}" ]; then - print_err_msg_exit "\ -Forecast length is greater than maximum allowed length: - FCST_LEN_HRS = ${FCST_LEN_HRS} - fcst_len_hrs_max = ${fcst_len_hrs_max}" -fi -# -#----------------------------------------------------------------------- -# -# Check whether the forecast length (FCST_LEN_HRS) is evenly divisible -# by the BC update interval (LBC_SPEC_INTVL_HRS). If not, print out a -# warning and exit this script. If so, generate an array of forecast -# hours at which the boundary values will be updated. -# -#----------------------------------------------------------------------- -# -rem=$(( ${FCST_LEN_HRS}%${LBC_SPEC_INTVL_HRS} )) - -if [ "$rem" -ne "0" ]; then - print_err_msg_exit "\ -The forecast length (FCST_LEN_HRS) is not evenly divisible by the lateral -boundary conditions update interval (LBC_SPEC_INTVL_HRS): - FCST_LEN_HRS = ${FCST_LEN_HRS} - LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} - rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" -fi -# -#----------------------------------------------------------------------- -# -# Set the array containing the forecast hours at which the lateral -# boundary conditions (LBCs) need to be updated. Note that this array -# does not include the 0-th hour (initial time). -# -#----------------------------------------------------------------------- -# -LBC_SPEC_FCST_HRS=($( seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} \ - ${FCST_LEN_HRS} )) -# -#----------------------------------------------------------------------- -# -# If PREDEF_GRID_NAME is set to a non-empty string, set or reset native -# and write-component grid parameters according to the specified predefined -# domain. -# -#----------------------------------------------------------------------- -# -if [ ! -z "${PREDEF_GRID_NAME}" ]; then - - set_predef_grid_params \ - predef_grid_name="${PREDEF_GRID_NAME}" \ - dt_atmos="${DT_ATMOS}" \ - layout_x="${LAYOUT_X}" \ - layout_y="${LAYOUT_Y}" \ - blocksize="${BLOCKSIZE}" \ - quilting="${QUILTING}" \ - outvarname_grid_gen_method="GRID_GEN_METHOD" \ - outvarname_esggrid_lon_ctr="ESGgrid_LON_CTR" \ - outvarname_esggrid_lat_ctr="ESGgrid_LAT_CTR" \ - outvarname_esggrid_delx="ESGgrid_DELX" \ - outvarname_esggrid_dely="ESGgrid_DELY" \ - outvarname_esggrid_nx="ESGgrid_NX" \ - outvarname_esggrid_ny="ESGgrid_NY" \ - outvarname_esggrid_pazi="ESGgrid_PAZI" \ - outvarname_esggrid_wide_halo_width="ESGgrid_WIDE_HALO_WIDTH" \ - outvarname_gfdlgrid_lon_t6_ctr="GFDLgrid_LON_T6_CTR" \ - outvarname_gfdlgrid_lat_t6_ctr="GFDLgrid_LAT_T6_CTR" \ - outvarname_gfdlgrid_stretch_fac="GFDLgrid_STRETCH_FAC" \ - outvarname_gfdlgrid_num_cells="GFDLgrid_NUM_CELLS" \ - outvarname_gfdlgrid_refine_ratio="GFDLgrid_REFINE_RATIO" \ - outvarname_gfdlgrid_istart_of_rgnl_dom_on_t6g="GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G" \ - outvarname_gfdlgrid_iend_of_rgnl_dom_on_t6g="GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G" \ - outvarname_gfdlgrid_jstart_of_rgnl_dom_on_t6g="GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G" \ - outvarname_gfdlgrid_jend_of_rgnl_dom_on_t6g="GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G" \ - outvarname_gfdlgrid_use_num_cells_in_filenames="GFDLgrid_USE_NUM_CELLS_IN_FILENAMES" \ - outvarname_dt_atmos="DT_ATMOS" \ - outvarname_layout_x="LAYOUT_X" \ - outvarname_layout_y="LAYOUT_Y" \ - outvarname_blocksize="BLOCKSIZE" \ - outvarname_wrtcmp_write_groups="WRTCMP_write_groups" \ - outvarname_wrtcmp_write_tasks_per_group="WRTCMP_write_tasks_per_group" \ - outvarname_wrtcmp_output_grid="WRTCMP_output_grid" \ - outvarname_wrtcmp_cen_lon="WRTCMP_cen_lon" \ - outvarname_wrtcmp_cen_lat="WRTCMP_cen_lat" \ - outvarname_wrtcmp_stdlat1="WRTCMP_stdlat1" \ - outvarname_wrtcmp_stdlat2="WRTCMP_stdlat2" \ - outvarname_wrtcmp_nx="WRTCMP_nx" \ - outvarname_wrtcmp_ny="WRTCMP_ny" \ - outvarname_wrtcmp_lon_lwr_left="WRTCMP_lon_lwr_left" \ - outvarname_wrtcmp_lat_lwr_left="WRTCMP_lat_lwr_left" \ - outvarname_wrtcmp_lon_upr_rght="WRTCMP_lon_upr_rght" \ - outvarname_wrtcmp_lat_upr_rght="WRTCMP_lat_upr_rght" \ - outvarname_wrtcmp_dx="WRTCMP_dx" \ - outvarname_wrtcmp_dy="WRTCMP_dy" \ - outvarname_wrtcmp_dlon="WRTCMP_dlon" \ - outvarname_wrtcmp_dlat="WRTCMP_dlat" - -fi -# -#----------------------------------------------------------------------- -# -# Make sure GRID_GEN_METHOD is set to a valid value. -# -#----------------------------------------------------------------------- -# -err_msg="\ -The horizontal grid generation method specified in GRID_GEN_METHOD is -not supported: - GRID_GEN_METHOD = \"${GRID_GEN_METHOD}\"" -check_var_valid_value \ - "GRID_GEN_METHOD" "valid_vals_GRID_GEN_METHOD" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# For a "GFDLgrid" type of grid, make sure GFDLgrid_NUM_CELLS is set to -# a valid value. -# -#----------------------------------------------------------------------- -# -if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - err_msg="\ -The number of grid cells per tile in each horizontal direction specified -in GFDLgrid_NUM_CELLS is not supported: - GFDLgrid_NUM_CELLS = \"${GFDLgrid_NUM_CELLS}\"" - check_var_valid_value "GFDLgrid_NUM_CELLS" "valid_vals_GFDLgrid_NUM_CELLS" "${err_msg}" -fi -# -#----------------------------------------------------------------------- -# -# Check to make sure that various computational parameters needed by the -# forecast model are set to non-empty values. At this point in the -# experiment generation, all of these should be set to valid (non-empty) -# values. -# -#----------------------------------------------------------------------- -# -if [ -z "${DT_ATMOS}" ]; then - print_err_msg_exit "\ -The forecast model main time step (DT_ATMOS) is set to a null string: - DT_ATMOS = ${DT_ATMOS} -Please set this to a valid numerical value in the user-specified experiment -configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" -fi - -if [ -z "${LAYOUT_X}" ]; then - print_err_msg_exit "\ -The number of MPI processes to be used in the x direction (LAYOUT_X) by -the forecast job is set to a null string: - LAYOUT_X = ${LAYOUT_X} -Please set this to a valid numerical value in the user-specified experiment -configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" -fi - -if [ -z "${LAYOUT_Y}" ]; then - print_err_msg_exit "\ -The number of MPI processes to be used in the y direction (LAYOUT_Y) by -the forecast job is set to a null string: - LAYOUT_Y = ${LAYOUT_Y} -Please set this to a valid numerical value in the user-specified experiment -configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" -fi - -if [ -z "${BLOCKSIZE}" ]; then - print_err_msg_exit "\ -The cache size to use for each MPI task of the forecast (BLOCKSIZE) is -set to a null string: - BLOCKSIZE = ${BLOCKSIZE} -Please set this to a valid numerical value in the user-specified experiment -configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"${EXPT_CONFIG_FP}\"" -fi -# -#----------------------------------------------------------------------- -# -# If performing sub-hourly model output and post-processing, check that -# the output interval DT_SUBHOURLY_POST_MNTS (in minutes) is specified -# correctly. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "SUB_HOURLY_POST" "valid_vals_BOOLEAN" -SUB_HOURLY_POST=$(boolify "${SUB_HOURLY_POST}") - -if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then -# -# Check that DT_SUBHOURLY_POST_MNTS is a string consisting of one or two -# digits. -# - mnts_or_null=$( printf "%s" "${DT_SUBHOURLY_POST_MNTS}" | \ - $SED -n -r -e "s/^([0-9])([0-9])?$/\1\2/p" ) - if [ -z "${mnts_or_null}" ]; then - print_err_msg_exit "\ -When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), -DT_SUBHOURLY_POST_MNTS must be set to a one- or two-digit integer but -in this case is not: - SUB_HOURLY_POST = \"${SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"${DT_SUBHOURLY_POST_MNTS}\"" - fi -# -# Check that DT_SUBHOURLY_POST_MNTS is between 0 and 59, inclusive. -# - if [ ${DT_SUBHOURLY_POST_MNTS} -lt "0" ] || \ - [ ${DT_SUBHOURLY_POST_MNTS} -gt "59" ]; then - print_err_msg_exit "\ -When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), -DT_SUBHOURLY_POST_MNTS must be set to an integer between 0 and 59, -inclusive but in this case is not: - SUB_HOURLY_POST = \"${SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"${DT_SUBHOURLY_POST_MNTS}\"" - fi -# -# Check that DT_SUBHOURLY_POST_MNTS (after converting to seconds) is -# evenly divisible by the forecast model's main time step DT_ATMOS. -# - rem=$(( DT_SUBHOURLY_POST_MNTS*60 % DT_ATMOS )) - if [ ${rem} -ne 0 ]; then - print_err_msg_exit "\ -When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), -the time interval specified by DT_SUBHOURLY_POST_MNTS (after converting -to seconds) must be evenly divisible by the time step DT_ATMOS used in -the forecast model, i.e. the remainder (rem) must be zero. In this case, -it is not: - SUB_HOURLY_POST = \"${SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"${DT_SUBHOURLY_POST_MNTS}\" - DT_ATMOS = \"${DT_ATMOS}\" - rem = \$(( (DT_SUBHOURLY_POST_MNTS*60) %% DT_ATMOS )) = $rem -Please reset DT_SUBHOURLY_POST_MNTS and/or DT_ATMOS so that this remainder -is zero." - fi -# -# If DT_SUBHOURLY_POST_MNTS is set to 0 (with SUB_HOURLY_POST set to -# "TRUE"), then we're not really performing subhourly post-processing. -# In this case, reset SUB_HOURLY_POST to "FALSE" and print out an -# informational message that such a change was made. -# - if [ "${DT_SUBHOURLY_POST_MNTS}" -eq "0" ]; then - print_info_msg "\ -When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), -DT_SUBHOURLY_POST_MNTS must be set to a value greater than 0; otherwise, -sub-hourly output is not really being performed: - SUB_HOURLY_POST = \"${SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"${DT_SUBHOURLY_POST_MNTS}\" -Resetting SUB_HOURLY_POST to \"FALSE\". If you do not want this, you -must set DT_SUBHOURLY_POST_MNTS to something other than zero." - SUB_HOURLY_POST="FALSE" - fi -# -# For now, the sub-hourly capability is restricted to having values of -# DT_SUBHOURLY_POST_MNTS that evenly divide into 60 minutes. This is -# because the jinja rocoto XML template (${WFLOW_XML_FN}) assumes that -# model output is generated at the top of every hour (i.e. at 00 minutes). -# This restricts DT_SUBHOURLY_POST_MNTS to the following values (inluding -# both cases with and without a leading 0): -# -# "1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30" -# -# This restriction will be removed in a future version of the workflow, -# For now, check that DT_SUBHOURLY_POST_MNTS is one of the above values. -# - if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then - check_var_valid_value "DT_SUBHOURLY_POST_MNTS" "valid_vals_DT_SUBHOURLY_POST_MNTS" - fi - -fi -# -#----------------------------------------------------------------------- -# -# If the base directory (EXPT_BASEDIR) in which the experiment subdirectory -# (EXPT_SUBDIR) will be located does not start with a "/", then it is -# either set to a null string or contains a relative directory. In both -# cases, prepend to it the absolute path of the default directory under -# which the experiment directories are placed. If EXPT_BASEDIR was set -# to a null string, it will get reset to this default experiment directory, -# and if it was set to a relative directory, it will get reset to an -# absolute directory that points to the relative directory under the -# default experiment directory. Then create EXPT_BASEDIR if it doesn't -# already exist. -# -#----------------------------------------------------------------------- -# -if [ "${EXPT_BASEDIR:0:1}" != "/" ]; then - EXPT_BASEDIR="${SR_WX_APP_TOP_DIR}/../expt_dirs/${EXPT_BASEDIR}" -fi -EXPT_BASEDIR="$( $READLINK -m ${EXPT_BASEDIR} )" -mkdir_vrfy -p "${EXPT_BASEDIR}" -# -#----------------------------------------------------------------------- -# -# If the experiment subdirectory name (EXPT_SUBDIR) is set to an empty -# string, print out an error message and exit. -# -#----------------------------------------------------------------------- -# -if [ -z "${EXPT_SUBDIR}" ]; then - print_err_msg_exit "\ -The name of the experiment subdirectory (EXPT_SUBDIR) cannot be empty: - EXPT_SUBDIR = \"${EXPT_SUBDIR}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set the full path to the experiment directory. Then check if it already -# exists and if so, deal with it as specified by PREEXISTING_DIR_METHOD. -# -#----------------------------------------------------------------------- -# -EXPTDIR="${EXPT_BASEDIR}/${EXPT_SUBDIR}" -check_for_preexist_dir_file "$EXPTDIR" "${PREEXISTING_DIR_METHOD}" -# -#----------------------------------------------------------------------- -# -# Set other directories, some of which may depend on EXPTDIR (depending -# on whether we're running in NCO or community mode, i.e. whether RUN_ENVIR -# is set to "nco" or "community"). Definitions: -# -# LOGDIR: -# Directory in which the log files from the workflow tasks will be placed. -# -# FIXam: -# This is the directory that will contain the fixed files or symlinks to -# the fixed files containing various fields on global grids (which are -# usually much coarser than the native FV3-LAM grid). -# -# FIXclim: -# This is the directory that will contain the MERRA2 aerosol climatology -# data file and lookup tables for optics properties -# -# FIXLAM: -# This is the directory that will contain the fixed files or symlinks to -# the fixed files containing the grid, orography, and surface climatology -# on the native FV3-LAM grid. -# -# CYCLE_BASEDIR: -# The base directory in which the directories for the various cycles will -# be placed. -# -# COMROOT: -# In NCO mode, this is the full path to the "com" directory under which -# output from the RUN_POST_TN task will be placed. Note that this output -# is not placed directly under COMROOT but several directories further -# down. More specifically, for a cycle starting at yyyymmddhh, it is at -# -# $COMROOT/$NET/$model_ver/$RUN.$yyyymmdd/$hh -# -# Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMOROOT -# is not used by the workflow in community mode. -# -# COMOUT_BASEDIR: -# In NCO mode, this is the base directory directly under which the output -# from the RUN_POST_TN task will be placed, i.e. it is the cycle-independent -# portion of the RUN_POST_TN task's output directory. It is given by -# -# $COMROOT/$NET/$model_ver -# -# COMOUT_BASEDIR is not used by the workflow in community mode. -# -#----------------------------------------------------------------------- -# -LOGDIR="${EXPTDIR}/log" - -FIXam="${EXPTDIR}/fix_am" -FIXclim="${EXPTDIR}/fix_clim" -FIXLAM="${EXPTDIR}/fix_lam" - -if [ "${RUN_ENVIR}" = "nco" ]; then - CYCLE_BASEDIR="${STMP}/tmpnwprd/${RUN}" - check_for_preexist_dir_file "${CYCLE_BASEDIR}" "${PREEXISTING_DIR_METHOD}" - COMROOT="${PTMP}/com" - COMOUT_BASEDIR="${COMROOT}/${NET}/${model_ver}" - check_for_preexist_dir_file "${COMOUT_BASEDIR}" "${PREEXISTING_DIR_METHOD}" -else - CYCLE_BASEDIR="$EXPTDIR" - COMROOT="" - COMOUT_BASEDIR="" -fi -# -#----------------------------------------------------------------------- -# -# -# If POST_OUTPUT_DOMAIN_NAME has not been specified by the user, set it -# to PREDEF_GRID_NAME (which won't be empty if using a predefined grid). -# Then change it to lowercase. Finally, ensure that it does not end up -# getting set to an empty string. -# -#----------------------------------------------------------------------- -# -POST_OUTPUT_DOMAIN_NAME="${POST_OUTPUT_DOMAIN_NAME:-${PREDEF_GRID_NAME}}" -POST_OUTPUT_DOMAIN_NAME=$(echo_lowercase ${POST_OUTPUT_DOMAIN_NAME}) - -if [ -z "${POST_OUTPUT_DOMAIN_NAME}" ]; then - print_err_msg_exit "\ -The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) -has not been set: - POST_OUTPUT_DOMAIN_NAME = \"${POST_OUTPUT_DOMAIN_NAME}\" -If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME -is set to a null string), POST_OUTPUT_DOMAIN_NAME must be set in the SRW -App's configuration file (\"${EXPT_CONFIG_FN}\")." -fi -# -#----------------------------------------------------------------------- -# -# The FV3 forecast model needs the following input files in the run -# directory to start a forecast: -# -# (1) The data table file -# (2) The diagnostics table file -# (3) The field table file -# (4) The FV3 namelist file -# (5) The model configuration file -# (6) The NEMS configuration file -# (7) The CCPP physics suite definition file -# -# The workflow contains templates for the first six of these files. -# Template files are versions of these files that contain placeholder -# (i.e. dummy) values for various parameters. The experiment generation -# and/or the forecast task (i.e. J-job) scripts copy these templates to -# appropriate locations in the experiment directory (e.g. to the top of -# the experiment directory, to one of the cycle subdirectories, etc) and -# replace the placeholders with actual values to obtain the files that -# are used as inputs to the forecast model. -# -# Note that the CCPP physics suite defintion file (SDF) does not have a -# corresponding template file because it does not contain any values -# that need to be replaced according to the experiment configuration. -# This file simply needs to be copied over from its location in the -# forecast model's directory structure to the experiment directory. -# -# Below, we first set the names of the templates for the first six files -# listed above. We then set the full paths to these template files. -# Note that some of these file names depend on the physics suite while -# others do not. -# -#----------------------------------------------------------------------- -# -dot_ccpp_phys_suite_or_null=".${CCPP_PHYS_SUITE}" - -# Names of input files that the forecast model (ufs-weather-model) expects -# to read in. These should only be changed if the input file names in the -# forecast model code are changed. -#---------------------------------- -DATA_TABLE_FN="data_table" -DIAG_TABLE_FN="diag_table" -FIELD_TABLE_FN="field_table" -MODEL_CONFIG_FN="model_configure" -NEMS_CONFIG_FN="nems.configure" -#---------------------------------- - -DATA_TABLE_TMPL_FN="${DATA_TABLE_TMPL_FN:-${DATA_TABLE_FN}}" -DIAG_TABLE_TMPL_FN="${DIAG_TABLE_TMPL_FN:-${DIAG_TABLE_FN}}${dot_ccpp_phys_suite_or_null}" -FIELD_TABLE_TMPL_FN="${FIELD_TABLE_TMPL_FN:-${FIELD_TABLE_FN}}${dot_ccpp_phys_suite_or_null}" -MODEL_CONFIG_TMPL_FN="${MODEL_CONFIG_TMPL_FN:-${MODEL_CONFIG_FN}}" -NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_TMPL_FN:-${NEMS_CONFIG_FN}}" - -DATA_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DATA_TABLE_TMPL_FN}" -DIAG_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DIAG_TABLE_TMPL_FN}" -FIELD_TABLE_TMPL_FP="${TEMPLATE_DIR}/${FIELD_TABLE_TMPL_FN}" -FV3_NML_BASE_SUITE_FP="${TEMPLATE_DIR}/${FV3_NML_BASE_SUITE_FN}" -FV3_NML_YAML_CONFIG_FP="${TEMPLATE_DIR}/${FV3_NML_YAML_CONFIG_FN}" -FV3_NML_BASE_ENS_FP="${EXPTDIR}/${FV3_NML_BASE_ENS_FN}" -MODEL_CONFIG_TMPL_FP="${TEMPLATE_DIR}/${MODEL_CONFIG_TMPL_FN}" -NEMS_CONFIG_TMPL_FP="${TEMPLATE_DIR}/${NEMS_CONFIG_TMPL_FN}" -# -#----------------------------------------------------------------------- -# -# Set: -# -# 1) the variable CCPP_PHYS_SUITE_FN to the name of the CCPP physics -# suite definition file. -# 2) the variable CCPP_PHYS_SUITE_IN_CCPP_FP to the full path of this -# file in the forecast model's directory structure. -# 3) the variable CCPP_PHYS_SUITE_FP to the full path of this file in -# the experiment directory. -# -# Note that the experiment/workflow generation scripts will copy this -# file from CCPP_PHYS_SUITE_IN_CCPP_FP to CCPP_PHYS_SUITE_FP. Then, for -# each cycle, the forecast launch script will create a link in the cycle -# run directory to the copy of this file at CCPP_PHYS_SUITE_FP. -# -#----------------------------------------------------------------------- -# -CCPP_PHYS_SUITE_FN="suite_${CCPP_PHYS_SUITE}.xml" -CCPP_PHYS_SUITE_IN_CCPP_FP="${UFS_WTHR_MDL_DIR}/FV3/ccpp/suites/${CCPP_PHYS_SUITE_FN}" -CCPP_PHYS_SUITE_FP="${EXPTDIR}/${CCPP_PHYS_SUITE_FN}" -if [ ! -f "${CCPP_PHYS_SUITE_IN_CCPP_FP}" ]; then - print_err_msg_exit "\ -The CCPP suite definition file (CCPP_PHYS_SUITE_IN_CCPP_FP) does not exist -in the local clone of the ufs-weather-model: - CCPP_PHYS_SUITE_IN_CCPP_FP = \"${CCPP_PHYS_SUITE_IN_CCPP_FP}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set: -# -# 1) the variable FIELD_DICT_FN to the name of the field dictionary -# file. -# 2) the variable FIELD_DICT_IN_UWM_FP to the full path of this -# file in the forecast model's directory structure. -# 3) the variable FIELD_DICT_FP to the full path of this file in -# the experiment directory. -# -#----------------------------------------------------------------------- -# -FIELD_DICT_FN="fd_nems.yaml" -FIELD_DICT_IN_UWM_FP="${UFS_WTHR_MDL_DIR}/tests/parm/${FIELD_DICT_FN}" -FIELD_DICT_FP="${EXPTDIR}/${FIELD_DICT_FN}" -if [ ! -f "${FIELD_DICT_IN_UWM_FP}" ]; then - print_err_msg_exit "\ -The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist -in the local clone of the ufs-weather-model: - FIELD_DICT_IN_UWM_FP = \"${FIELD_DICT_IN_UWM_FP}\"" -fi -# -#----------------------------------------------------------------------- -# -# Call the function that sets the ozone parameterization being used and -# modifies associated parameters accordingly. -# -#----------------------------------------------------------------------- -# -set_ozone_param \ - ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ - output_varname_ozone_param="OZONE_PARAM" -# -#----------------------------------------------------------------------- -# -# Set the full paths to those forecast model input files that are cycle- -# independent, i.e. they don't include information about the cycle's -# starting day/time. These are: -# -# * The data table file [(1) in the list above)] -# * The field table file [(3) in the list above)] -# * The FV3 namelist file [(4) in the list above)] -# * The NEMS configuration file [(6) in the list above)] -# -# Since they are cycle-independent, the experiment/workflow generation -# scripts will place them in the main experiment directory (EXPTDIR). -# The script that runs each cycle will then create links to these files -# in the run directories of the individual cycles (which are subdirecto- -# ries under EXPTDIR). -# -# The remaining two input files to the forecast model, i.e. -# -# * The diagnostics table file [(2) in the list above)] -# * The model configuration file [(5) in the list above)] -# -# contain parameters that depend on the cycle start date. Thus, custom -# versions of these two files must be generated for each cycle and then -# placed directly in the run directories of the cycles (not EXPTDIR). -# For this reason, the full paths to their locations vary by cycle and -# cannot be set here (i.e. they can only be set in the loop over the -# cycles in the rocoto workflow XML file). -# -#----------------------------------------------------------------------- -# -DATA_TABLE_FP="${EXPTDIR}/${DATA_TABLE_FN}" -FIELD_TABLE_FP="${EXPTDIR}/${FIELD_TABLE_FN}" -FV3_NML_FN="${FV3_NML_BASE_SUITE_FN%.*}" -FV3_NML_FP="${EXPTDIR}/${FV3_NML_FN}" -NEMS_CONFIG_FP="${EXPTDIR}/${NEMS_CONFIG_FN}" -# -#----------------------------------------------------------------------- -# -# If USE_USER_STAGED_EXTRN_FILES is set to TRUE, make sure that the user- -# specified directories under which the external model files should be -# located actually exist. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_BOOLEAN" -USE_USER_STAGED_EXTRN_FILES=$(boolify "${USE_USER_STAGED_EXTRN_FILES}") - -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - - # Check for the base directory up to the first templated field. - if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_ICS%%\$*})" ]; then - print_err_msg_exit "\ -The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged -external model files for generating ICs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_ICS = \"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\"" - fi - - if [ ! -d "$(dirname ${EXTRN_MDL_SOURCE_BASEDIR_LBCS%%\$*})" ]; then - print_err_msg_exit "\ -The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged -external model files for generating LBCs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_LBCS = \"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\"" - fi - -fi - -check_var_valid_value "NOMADS" "valid_vals_BOOLEAN" -NOMADS=$(boolify "${NOMADS}") -# -#----------------------------------------------------------------------- -# -# Make sure that DO_ENSEMBLE is set to a valid value. Then set the names -# of the ensemble members. These will be used to set the ensemble member -# directories. Also, set the full path to the FV3 namelist file corresponding -# to each ensemble member. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_ENSEMBLE" "valid_vals_BOOLEAN" -DO_ENSEMBLE=$(boolify "${DO_ENSEMBLE}") - -NDIGITS_ENSMEM_NAMES="0" -ENSMEM_NAMES=("") -FV3_NML_ENSMEM_FPS=("") -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - NDIGITS_ENSMEM_NAMES="${#NUM_ENS_MEMBERS}" -# Strip away all leading zeros in NUM_ENS_MEMBERS by converting it to a -# decimal (leading zeros will cause bash to interpret the number as an -# octal). Note that the variable definitions file will therefore contain -# the version of NUM_ENS_MEMBERS with any leading zeros stripped away. - NUM_ENS_MEMBERS="$((10#${NUM_ENS_MEMBERS}))" - fmt="%0${NDIGITS_ENSMEM_NAMES}d" - for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - ip1=$( printf "$fmt" $((i+1)) ) - ENSMEM_NAMES[$i]="mem${ip1}" - FV3_NML_ENSMEM_FPS[$i]="$EXPTDIR/${FV3_NML_FN}_${ENSMEM_NAMES[$i]}" - done -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. -# -#----------------------------------------------------------------------- -# -if [ "${DO_ENSEMBLE}" = "FALSE" ] && [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" -o \ - "${RUN_TASK_VX_ENSPOINT}" = "TRUE" ]; then - print_err_msg_exit "\ -Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"${DO_ENSEMBLE}\" - RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" - RUN_TASK_VX_ENSPOINT = \"${RUN_TASK_VX_ENSPOINT}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set the full path to the forecast model executable. -# -#----------------------------------------------------------------------- -# -FV3_EXEC_FP="${EXECDIR}/${FV3_EXEC_FN}" -# -#----------------------------------------------------------------------- -# -# Set the full path to the script that can be used to (re)launch the -# workflow. Also, if USE_CRON_TO_RELAUNCH is set to TRUE, set the line -# to add to the cron table to automatically relaunch the workflow every -# CRON_RELAUNCH_INTVL_MNTS minutes. Otherwise, set the variable con- -# taining this line to a null string. -# -#----------------------------------------------------------------------- -# -WFLOW_LAUNCH_SCRIPT_FP="$USHDIR/${WFLOW_LAUNCH_SCRIPT_FN}" -WFLOW_LAUNCH_LOG_FP="$EXPTDIR/${WFLOW_LAUNCH_LOG_FN}" -if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then - CRONTAB_LINE="*/${CRON_RELAUNCH_INTVL_MNTS} * * * * cd $EXPTDIR && \ -./${WFLOW_LAUNCH_SCRIPT_FN} called_from_cron=\"TRUE\" >> ./${WFLOW_LAUNCH_LOG_FN} 2>&1" -else - CRONTAB_LINE="" -fi -# -#----------------------------------------------------------------------- -# -# Set the full path to the script that, for a given task, loads the -# necessary module files and runs the tasks. -# -#----------------------------------------------------------------------- -# -LOAD_MODULES_RUN_TASK_FP="$USHDIR/load_modules_run_task.sh" -# -#----------------------------------------------------------------------- -# -# Define the various work subdirectories under the main work directory. -# Each of these corresponds to a different step/substep/task in the pre- -# processing, as follows: -# -# GRID_DIR: -# Directory in which the grid files will be placed (if RUN_TASK_MAKE_GRID -# is set to "TRUE") or searched for (if RUN_TASK_MAKE_GRID is set to -# "FALSE"). -# -# OROG_DIR: -# Directory in which the orography files will be placed (if RUN_TASK_MAKE_OROG -# is set to "TRUE") or searched for (if RUN_TASK_MAKE_OROG is set to -# "FALSE"). -# -# SFC_CLIMO_DIR: -# Directory in which the surface climatology files will be placed (if -# RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE") or searched for (if -# RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE"). -# -#---------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - nco_fix_dir="${DOMAIN_PREGEN_BASEDIR}/${PREDEF_GRID_NAME}" - if [ ! -d "${nco_fix_dir}" ]; then - print_err_msg_exit "\ -The directory (nco_fix_dir) that should contain the pregenerated grid, -orography, and surface climatology files does not exist: - nco_fix_dir = \"${nco_fix_dir}\"" - fi - - if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_GRID}" = "FALSE" -a \ - "${GRID_DIR}" != "${nco_fix_dir}" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated -grid files already exist in the directory - - \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} - -where - - DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" - -Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must -be set to \"FALSE\"), and the directory in which to look for the grid -files (i.e. GRID_DIR) must be set to the one above. Current values for -these quantities are: - - RUN_TASK_MAKE_GRID = \"${RUN_TASK_MAKE_GRID}\" - GRID_DIR = \"${GRID_DIR}\" - -Resetting RUN_TASK_MAKE_GRID to \"FALSE\" and GRID_DIR to the one above. -Reset values are: -" - - RUN_TASK_MAKE_GRID="FALSE" - GRID_DIR="${nco_fix_dir}" - - msg="$msg"" - RUN_TASK_MAKE_GRID = \"${RUN_TASK_MAKE_GRID}\" - GRID_DIR = \"${GRID_DIR}\" -" - - print_info_msg "$msg" - - fi - - if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_OROG}" = "FALSE" -a \ - "${OROG_DIR}" != "${nco_fix_dir}" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated -orography files already exist in the directory - \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} - -where - - DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" - -Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must -be set to \"FALSE\"), and the directory in which to look for the orography -files (i.e. OROG_DIR) must be set to the one above. Current values for -these quantities are: - - RUN_TASK_MAKE_OROG = \"${RUN_TASK_MAKE_OROG}\" - OROG_DIR = \"${OROG_DIR}\" - -Resetting RUN_TASK_MAKE_OROG to \"FALSE\" and OROG_DIR to the one above. -Reset values are: -" - - RUN_TASK_MAKE_OROG="FALSE" - OROG_DIR="${nco_fix_dir}" - - msg="$msg"" - RUN_TASK_MAKE_OROG = \"${RUN_TASK_MAKE_OROG}\" - OROG_DIR = \"${OROG_DIR}\" -" - - print_info_msg "$msg" - - fi - - if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" -a \ - "${SFC_CLIMO_DIR}" != "${nco_fix_dir}" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated -surface climatology files already exist in the directory - - \${DOMAIN_PREGEN_BASEDIR}/\${PREDEF_GRID_NAME} - -where - - DOMAIN_PREGEN_BASEDIR = \"${DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"${PREDEF_GRID_NAME}\" - -Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO -must be set to \"FALSE\"), and the directory in which to look for the -surface climatology files (i.e. SFC_CLIMO_DIR) must be set to the one -above. Current values for these quantities are: - - RUN_TASK_MAKE_SFC_CLIMO = \"${RUN_TASK_MAKE_SFC_CLIMO}\" - SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\" - -Resetting RUN_TASK_MAKE_SFC_CLIMO to \"FALSE\" and SFC_CLIMO_DIR to the -one above. Reset values are: -" - - RUN_TASK_MAKE_SFC_CLIMO="FALSE" - SFC_CLIMO_DIR="${nco_fix_dir}" - - msg="$msg"" - RUN_TASK_MAKE_SFC_CLIMO = \"${RUN_TASK_MAKE_SFC_CLIMO}\" - SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\" -" - - print_info_msg "$msg" - - fi - - if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that the verification -will not be run. - RUN_TASK_VX_GRIDSTAT = \"${RUN_TASK_VX_GRIDSTAT}\" -Resetting RUN_TASK_VX_GRIDSTAT to \"FALSE\" -Reset value is:" - - RUN_TASK_VX_GRIDSTAT="FALSE" - - msg="$msg"" - RUN_TASK_VX_GRIDSTAT = \"${RUN_TASK_VX_GRIDSTAT}\" -" - - print_info_msg "$msg" - - fi - - if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that the verification -will not be run. - RUN_TASK_VX_POINTSTAT = \"${RUN_TASK_VX_POINTSTAT}\" -Resetting RUN_TASK_VX_POINTSTAT to \"FALSE\" -Reset value is:" - - RUN_TASK_VX_POINTSTAT="FALSE" - - msg="$msg"" - RUN_TASK_VX_POINTSTAT = \"${RUN_TASK_VX_POINTSTAT}\" -" - - print_info_msg "$msg" - - fi - - if [ "${RUN_TASK_VX_ENSGRID}" = "TRUE" ]; then - - msg=" -When RUN_ENVIR is set to \"nco\", it is assumed that the verification -will not be run. - RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" -Resetting RUN_TASK_VX_ENSGRID to \"FALSE\" -Reset value is:" - - RUN_TASK_VX_ENSGRID="FALSE" - - msg="$msg"" - RUN_TASK_VX_ENSGRID = \"${RUN_TASK_VX_ENSGRID}\" -" - - print_info_msg "$msg" - - fi -# -#----------------------------------------------------------------------- -# -# Now consider community mode. -# -#----------------------------------------------------------------------- -# -else -# -# If RUN_TASK_MAKE_GRID is set to "FALSE", the workflow will look for -# the pregenerated grid files in GRID_DIR. In this case, make sure that -# GRID_DIR exists. Otherwise, set it to a predefined location under the -# experiment directory (EXPTDIR). -# - if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - if [ ! -d "${GRID_DIR}" ]; then - print_err_msg_exit "\ -The directory (GRID_DIR) that should contain the pregenerated grid files -does not exist: - GRID_DIR = \"${GRID_DIR}\"" - fi - else - GRID_DIR="$EXPTDIR/grid" - fi -# -# If RUN_TASK_MAKE_OROG is set to "FALSE", the workflow will look for -# the pregenerated orography files in OROG_DIR. In this case, make sure -# that OROG_DIR exists. Otherwise, set it to a predefined location under -# the experiment directory (EXPTDIR). -# - if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then - if [ ! -d "${OROG_DIR}" ]; then - print_err_msg_exit "\ -The directory (OROG_DIR) that should contain the pregenerated orography -files does not exist: - OROG_DIR = \"${OROG_DIR}\"" - fi - else - OROG_DIR="$EXPTDIR/orog" - fi -# -# If RUN_TASK_MAKE_SFC_CLIMO is set to "FALSE", the workflow will look -# for the pregenerated surface climatology files in SFC_CLIMO_DIR. In -# this case, make sure that SFC_CLIMO_DIR exists. Otherwise, set it to -# a predefined location under the experiment directory (EXPTDIR). -# - if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - if [ ! -d "${SFC_CLIMO_DIR}" ]; then - print_err_msg_exit "\ -The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface -climatology files does not exist: - SFC_CLIMO_DIR = \"${SFC_CLIMO_DIR}\"" - fi - else - SFC_CLIMO_DIR="$EXPTDIR/sfc_climo" - fi - -fi -# -#----------------------------------------------------------------------- -# -# Make sure EXTRN_MDL_NAME_ICS is set to a valid value. -# -#----------------------------------------------------------------------- -# -err_msg="\ -The external model specified in EXTRN_MDL_NAME_ICS that provides initial -conditions (ICs) and surface fields to the FV3-LAM is not supported: - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" -check_var_valid_value \ - "EXTRN_MDL_NAME_ICS" "valid_vals_EXTRN_MDL_NAME_ICS" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# Make sure EXTRN_MDL_NAME_LBCS is set to a valid value. -# -#----------------------------------------------------------------------- -# -err_msg="\ -The external model specified in EXTRN_MDL_NAME_ICS that provides lateral -boundary conditions (LBCs) to the FV3-LAM is not supported: - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" -check_var_valid_value \ - "EXTRN_MDL_NAME_LBCS" "valid_vals_EXTRN_MDL_NAME_LBCS" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# Make sure FV3GFS_FILE_FMT_ICS is set to a valid value. -# -#----------------------------------------------------------------------- -# -if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - err_msg="\ -The file format for FV3GFS external model files specified in FV3GFS_- -FILE_FMT_ICS is not supported: - FV3GFS_FILE_FMT_ICS = \"${FV3GFS_FILE_FMT_ICS}\"" - check_var_valid_value \ - "FV3GFS_FILE_FMT_ICS" "valid_vals_FV3GFS_FILE_FMT_ICS" "${err_msg}" -fi -# -#----------------------------------------------------------------------- -# -# Make sure FV3GFS_FILE_FMT_LBCS is set to a valid value. -# -#----------------------------------------------------------------------- -# -if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - err_msg="\ -The file format for FV3GFS external model files specified in FV3GFS_- -FILE_FMT_LBCS is not supported: - FV3GFS_FILE_FMT_LBCS = \"${FV3GFS_FILE_FMT_LBCS}\"" - check_var_valid_value \ - "FV3GFS_FILE_FMT_LBCS" "valid_vals_FV3GFS_FILE_FMT_LBCS" "${err_msg}" -fi -# -#----------------------------------------------------------------------- -# -# Set cycle-independent parameters associated with the external models -# from which we will obtain the ICs and LBCs. -# -#----------------------------------------------------------------------- -# -. ./set_extrn_mdl_params.sh -# -#----------------------------------------------------------------------- -# -# Set parameters according to the type of horizontal grid generation -# method specified. First consider GFDL's global-parent-grid based -# method. -# -#----------------------------------------------------------------------- -# -if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - - set_gridparams_GFDLgrid \ - lon_of_t6_ctr="${GFDLgrid_LON_T6_CTR}" \ - lat_of_t6_ctr="${GFDLgrid_LAT_T6_CTR}" \ - res_of_t6g="${GFDLgrid_NUM_CELLS}" \ - stretch_factor="${GFDLgrid_STRETCH_FAC}" \ - refine_ratio_t6g_to_t7g="${GFDLgrid_REFINE_RATIO}" \ - istart_of_t7_on_t6g="${GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G}" \ - iend_of_t7_on_t6g="${GFDLgrid_IEND_OF_RGNL_DOM_ON_T6G}" \ - jstart_of_t7_on_t6g="${GFDLgrid_JSTART_OF_RGNL_DOM_ON_T6G}" \ - jend_of_t7_on_t6g="${GFDLgrid_JEND_OF_RGNL_DOM_ON_T6G}" \ - verbose="${VERBOSE}" \ - outvarname_lon_of_t7_ctr="LON_CTR" \ - outvarname_lat_of_t7_ctr="LAT_CTR" \ - outvarname_nx_of_t7_on_t7g="NX" \ - outvarname_ny_of_t7_on_t7g="NY" \ - outvarname_halo_width_on_t7g="NHW" \ - outvarname_stretch_factor="STRETCH_FAC" \ - outvarname_istart_of_t7_with_halo_on_t6sg="ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - outvarname_iend_of_t7_with_halo_on_t6sg="IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - outvarname_jstart_of_t7_with_halo_on_t6sg="JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ - outvarname_jend_of_t7_with_halo_on_t6sg="JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" -# -#----------------------------------------------------------------------- -# -# Now consider Jim Purser's map projection/grid generation method. -# -#----------------------------------------------------------------------- -# -elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - - set_gridparams_ESGgrid \ - lon_ctr="${ESGgrid_LON_CTR}" \ - lat_ctr="${ESGgrid_LAT_CTR}" \ - nx="${ESGgrid_NX}" \ - ny="${ESGgrid_NY}" \ - pazi="${ESGgrid_PAZI}" \ - halo_width="${ESGgrid_WIDE_HALO_WIDTH}" \ - delx="${ESGgrid_DELX}" \ - dely="${ESGgrid_DELY}" \ - outvarname_lon_ctr="LON_CTR" \ - outvarname_lat_ctr="LAT_CTR" \ - outvarname_nx="NX" \ - outvarname_ny="NY" \ - outvarname_pazi="PAZI" \ - outvarname_halo_width="NHW" \ - outvarname_stretch_factor="STRETCH_FAC" \ - outvarname_del_angle_x_sg="DEL_ANGLE_X_SG" \ - outvarname_del_angle_y_sg="DEL_ANGLE_Y_SG" \ - outvarname_neg_nx_of_dom_with_wide_halo="NEG_NX_OF_DOM_WITH_WIDE_HALO" \ - outvarname_neg_ny_of_dom_with_wide_halo="NEG_NY_OF_DOM_WITH_WIDE_HALO" - -fi -# -#----------------------------------------------------------------------- -# -# Create a new experiment directory. Note that at this point we are -# guaranteed that there is no preexisting experiment directory. For -# platforms with no workflow manager, we need to create LOGDIR as well, -# since it won't be created later at runtime. -# -#----------------------------------------------------------------------- -# -mkdir_vrfy -p "$EXPTDIR" -mkdir_vrfy -p "$LOGDIR" -# -#----------------------------------------------------------------------- -# -# If not running the MAKE_GRID_TN, MAKE_OROG_TN, and/or MAKE_SFC_CLIMO -# tasks, create symlinks under the FIXLAM directory to pregenerated grid, -# orography, and surface climatology files. In the process, also set -# RES_IN_FIXLAM_FILENAMES, which is the resolution of the grid (in units -# of number of grid points on an equivalent global uniform cubed-sphere -# grid) used in the names of the fixed files in the FIXLAM directory. -# -#----------------------------------------------------------------------- -# -mkdir_vrfy -p "$FIXLAM" -RES_IN_FIXLAM_FILENAMES="" -# -#----------------------------------------------------------------------- -# -# If the grid file generation task in the workflow is going to be skipped -# (because pregenerated files are available), create links in the FIXLAM -# directory to the pregenerated grid files. -# -#----------------------------------------------------------------------- -# -res_in_grid_fns="" -if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - - link_fix \ - verbose="$VERBOSE" \ - file_group="grid" \ - output_varname_res_in_filenames="res_in_grid_fns" || \ - print_err_msg_exit "\ -Call to function to create links to grid files failed." - - RES_IN_FIXLAM_FILENAMES="${res_in_grid_fns}" - -fi -# -#----------------------------------------------------------------------- -# -# If the orography file generation task in the workflow is going to be -# skipped (because pregenerated files are available), create links in -# the FIXLAM directory to the pregenerated orography files. -# -#----------------------------------------------------------------------- -# -res_in_orog_fns="" -if [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ]; then - - link_fix \ - verbose="$VERBOSE" \ - file_group="orog" \ - output_varname_res_in_filenames="res_in_orog_fns" || \ - print_err_msg_exit "\ -Call to function to create links to orography files failed." - - if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ - [ "${res_in_orog_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then - print_err_msg_exit "\ -The resolution extracted from the orography file names (res_in_orog_fns) -does not match the resolution in other groups of files already consi- -dered (RES_IN_FIXLAM_FILENAMES): - res_in_orog_fns = ${res_in_orog_fns} - RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" - else - RES_IN_FIXLAM_FILENAMES="${res_in_orog_fns}" - fi - -fi -# -#----------------------------------------------------------------------- -# -# If the surface climatology file generation task in the workflow is -# going to be skipped (because pregenerated files are available), create -# links in the FIXLAM directory to the pregenerated surface climatology -# files. -# -#----------------------------------------------------------------------- -# -res_in_sfc_climo_fns="" -if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ]; then - - link_fix \ - verbose="$VERBOSE" \ - file_group="sfc_climo" \ - output_varname_res_in_filenames="res_in_sfc_climo_fns" || \ - print_err_msg_exit "\ -Call to function to create links to surface climatology files failed." - - if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ] && \ - [ "${res_in_sfc_climo_fns}" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then - print_err_msg_exit "\ -The resolution extracted from the surface climatology file names (res_- -in_sfc_climo_fns) does not match the resolution in other groups of files -already considered (RES_IN_FIXLAM_FILENAMES): - res_in_sfc_climo_fns = ${res_in_sfc_climo_fns} - RES_IN_FIXLAM_FILENAMES = ${RES_IN_FIXLAM_FILENAMES}" - else - RES_IN_FIXLAM_FILENAMES="${res_in_sfc_climo_fns}" - fi - -fi -# -#----------------------------------------------------------------------- -# -# The variable CRES is needed in constructing various file names. If -# not running the make_grid task, we can set it here. Otherwise, it -# will get set to a valid value by that task. -# -#----------------------------------------------------------------------- -# -CRES="" -if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - CRES="C${RES_IN_FIXLAM_FILENAMES}" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that WRITE_DOPOST is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "WRITE_DOPOST" "valid_vals_BOOLEAN" -WRITE_DOPOST=$(boolify "${WRITE_DOPOST}") - -if [ "$WRITE_DOPOST" = "TRUE" ] ; then - -# Turn off run_post - RUN_TASK_RUN_POST="FALSE" - -# Check if SUB_HOURLY_POST is on - if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then - print_err_msg_exit "\ -SUB_HOURLY_POST is NOT available with Inline Post yet." - fi -fi - -check_var_valid_value "QUILTING" "valid_vals_BOOLEAN" -QUILTING=$(boolify "$QUILTING") - -check_var_valid_value "PRINT_ESMF" "valid_vals_BOOLEAN" -PRINT_ESMF=$(boolify "${PRINT_ESMF}") - -# -#----------------------------------------------------------------------- -# -# Calculate PE_MEMBER01. This is the number of MPI tasks used for the -# forecast, including those for the write component if QUILTING is set -# to "TRUE". -# -#----------------------------------------------------------------------- -# -PE_MEMBER01=$(( LAYOUT_X*LAYOUT_Y )) -if [ "$QUILTING" = "TRUE" ]; then - PE_MEMBER01=$(( ${PE_MEMBER01} + ${WRTCMP_write_groups}*${WRTCMP_write_tasks_per_group} )) -fi - -print_info_msg "$VERBOSE" " -The number of MPI tasks for the forecast (including those for the write -component if it is being used) are: - PE_MEMBER01 = ${PE_MEMBER01}" -# -#----------------------------------------------------------------------- -# -# If the write-component is going to be used to write output files to -# disk (i.e. if QUILTING is set to "TRUE"), make sure that the grid type -# used by the write-component (WRTCMP_output_grid) is set to a valid value. -# -#----------------------------------------------------------------------- -# -if [ "$QUILTING" = "TRUE" ]; then - err_msg="\ -The coordinate system used by the write-component output grid specified -in WRTCMP_output_grid is not supported: - WRTCMP_output_grid = \"${WRTCMP_output_grid}\"" - check_var_valid_value \ - "WRTCMP_output_grid" "valid_vals_WRTCMP_output_grid" "${err_msg}" -fi -# -#----------------------------------------------------------------------- -# -# Calculate the number of nodes (NNODES_RUN_FCST) to request from the job -# scheduler for the forecast task (RUN_FCST_TN). This is just PE_MEMBER01 -# dividied by the number of processes per node we want to request for this -# task (PPN_RUN_FCST), then rounded up to the nearest integer, i.e. -# -# NNODES_RUN_FCST = ceil(PE_MEMBER01/PPN_RUN_FCST) -# -# where ceil(...) is the ceiling function, i.e. it rounds its floating -# point argument up to the next larger integer. Since in bash, division -# of two integers returns a truncated integer, and since bash has no -# built-in ceil(...) function, we perform the rounding-up operation by -# adding the denominator (of the argument of ceil(...) above) minus 1 to -# the original numerator, i.e. by redefining NNODES_RUN_FCST to be -# -# NNODES_RUN_FCST = (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST -# -#----------------------------------------------------------------------- -# -NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) -# -#----------------------------------------------------------------------- -# -# Call the function that checks whether the RUC land surface model (LSM) -# is being called by the physics suite and sets the workflow variable -# SDF_USES_RUC_LSM to "TRUE" or "FALSE" accordingly. -# -#----------------------------------------------------------------------- -# -check_ruc_lsm \ - ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ - output_varname_sdf_uses_ruc_lsm="SDF_USES_RUC_LSM" -# -#----------------------------------------------------------------------- -# -# Set the name of the file containing aerosol climatology data that, if -# necessary, can be used to generate approximate versions of the aerosol -# fields needed by Thompson microphysics. This file will be used to -# generate such approximate aerosol fields in the ICs and LBCs if Thompson -# MP is included in the physics suite and if the exteranl model for ICs -# or LBCs does not already provide these fields. Also, set the full path -# to this file. -# -#----------------------------------------------------------------------- -# -THOMPSON_MP_CLIMO_FN="Thompson_MP_MONTHLY_CLIMO.nc" -THOMPSON_MP_CLIMO_FP="$FIXam/${THOMPSON_MP_CLIMO_FN}" -# -#----------------------------------------------------------------------- -# -# Call the function that, if the Thompson microphysics parameterization -# is being called by the physics suite, modifies certain workflow arrays -# to ensure that fixed files needed by this parameterization are copied -# to the FIXam directory and appropriate symlinks to them are created in -# the run directories. This function also sets the workflow variable -# SDF_USES_THOMPSON_MP that indicates whether Thompson MP is called by -# the physics suite. -# -#----------------------------------------------------------------------- -# -set_thompson_mp_fix_files \ - ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}" \ - thompson_mp_climo_fn="${THOMPSON_MP_CLIMO_FN}" \ - output_varname_sdf_uses_thompson_mp="SDF_USES_THOMPSON_MP" -# -#----------------------------------------------------------------------- -# -# Set the full path to the experiment's variable definitions file. This -# file will contain definitions of variables (in bash syntax) needed by -# the various scripts in the workflow. -# -#----------------------------------------------------------------------- -# -GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" -# -#----------------------------------------------------------------------- -# -# Get the list of constants and their values. The result is saved in -# the variable "constant_defns". This will be written to the experiment's -# variable defintions file later below. -# -#----------------------------------------------------------------------- -# -print_info_msg " -Creating list of constants..." - -get_bash_file_contents fp="$USHDIR/${CONSTANTS_FN}" \ - outvarname_contents="constant_defns" - -print_info_msg "$DEBUG" " -The variable \"constant_defns\" containing definitions of various -constants is set as follows: - -${constant_defns} -" -# -#----------------------------------------------------------------------- -# -# Get the list of primary experiment variables and their default values -# from the default experiment configuration file (EXPT_DEFAULT_CONFIG_FN). -# By "primary", we mean those variables that are defined in the default -# configuration file and can be reset in the user-specified experiment -# configuration file (EXPT_CONFIG_FN). The default values will be updated -# below to user-specified ones and the result saved in the experiment's -# variable definitions file. -# -#----------------------------------------------------------------------- -# -print_info_msg " -Creating list of default experiment variable definitions..." - -get_bash_file_contents fp="$USHDIR/${EXPT_DEFAULT_CONFIG_FN}" \ - outvarname_contents="default_var_defns" - -print_info_msg "$DEBUG" " -The variable \"default_var_defns\" containing default values of primary -experiment variables is set as follows: - -${default_var_defns} -" -# -#----------------------------------------------------------------------- -# -# Create a list of primary experiment variable definitions containing -# updated values. By "updated", we mean non-default values. Values -# may have been updated due to the presence of user-specified values in -# the experiment configuration file (EXPT_CONFIG_FN) or due to other -# considerations (e.g. resetting depending on the platform the App is -# running on). -# -#----------------------------------------------------------------------- -# -print_info_msg " -Creating lists of (updated) experiment variable definitions..." -# -# Set the flag that specifies whether or not array variables will be -# recorded in the variable definitions file on one line or one element -# per line. Then, if writing arrays one element per line (i.e. multiline), -# set an escaped-newline character that needs to be included after every -# element of each array as the newline character in order for sed to -# write the line properly. -# -multiline_arrays="TRUE" -#multiline_arrays="FALSE" -escbksl_nl_or_null="" -if [ "${multiline_arrays}" = "TRUE" ]; then - escbksl_nl_or_null='\\\n' -fi -# -# Loop through the lines in default_var_defns. Reset the value of the -# variable on each line to the updated value (e.g. to a user-specified -# value, as opposed to the default value). The updated list of variables -# and values will be saved in var_defns. -# -var_defns="" -while read crnt_line; do -# -# Try to obtain the name of the variable being set on the current line. -# This will be successful only if the line consists of one or more non- -# whitespace characters representing the name of a variable followed by -# an equal sign, followed by zero or more characters representing the -# value that the variable is being set to. (Recall that in generating -# the variable default_var_defns, leading spaces on each line were -# stripped out). -# - var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) -# -# If var_name is not empty, then a variable name was found on the current -# line in default_var_defns. -# - if [ ! -z ${var_name} ]; then - - print_info_msg "$DEBUG" " -var_name = \"${var_name}\"" -# -# If the variable specified in var_name is set in the current environment -# (to either an empty or non-empty string), get its value and save it in -# var_value. Note that -# -# ${!var_name+x} -# -# will retrun the string "x" if the variable specified in var_name is -# set (to either an empty or non-empty string), and it will return an -# empty string if the variable specified in var_name is unset (i.e. if -# it is undefined). -# - unset "var_value" - if [ ! -z "${!var_name+x}" ]; then -# -# The variable may be a scalar or an array. Thus, we first treat it as -# an array and obtain the number of elements that it contains. -# - array_name_at="${var_name}[@]" - array=("${!array_name_at}") - num_elems="${#array[@]}" -# -# Set var_value to the updated value of the current experiment variable. -# How this is done depends on whether the variable is a scalar or an -# array. -# -# If the variable contains only one element, then it is a scalar. (It -# could be a 1-element array, but for simplicity, we treat that case as -# a scalar.) In this case, we enclose its value in single quotes and -# save the result in var_value. No variable expansion should be -# happening from variables saved in the var_defns file. -# - if [ "${num_elems}" -eq 1 ]; then - - var_value="${!var_name}" - rhs="'${var_value}'" -# -# If the variable contains more than one element, then it is an array. -# In this case, we build var_value in two steps as follows: -# -# 1) Generate a string containing each element of the array in double -# quotes and followed by a space (and followed by an optional backslash -# and newline if multiline_arrays has been set to "TRUE"). -# -# 2) Place parentheses around the double-quoted list of array elements -# generated in the first step. Note that there is no need to put a -# space before the closing parenthesis because during step 1 above, -# a space has already been placed after the last array element. -# - else - - var_value="" - printf -v "var_value" "%s${escbksl_nl_or_null}" "" - for (( i=0; i<${num_elems}; i++ )); do - printf -v "var_value" "%s${escbksl_nl_or_null}" "${var_value}\"${array[$i]}\" " - done - rhs="( ${var_value})" - - fi -# -# If for some reason the variable specified in var_name is not set in -# the current environment (to either an empty or non-empty string), below -# we will still include it in the variable definitions file and simply -# set it to a null string. Thus, here, we set its value (var_value) to -# an empty string). In this case, we also issue an informational message. -# - else - - print_info_msg " -The variable specified by \"var_name\" is not set in the current environment: - var_name = \"${var_name}\" -Setting its value in the variable definitions file to an empty string." - - rhs="''" - - fi -# -# Set the line containing the variable's definition. Then add the line -# to the list of all variable definitions. -# - var_defn="${var_name}=$rhs" - printf -v "var_defns" "%s\n" "${var_defns}${var_defn}" -# -# If var_name is empty, then a variable name was not found on the current -# line in default_var_defns. In this case, print out a warning and move -# on to the next line. -# - else - - print_info_msg " -Could not extract a variable name from the current line in \"default_var_defns\" -(probably because it does not contain an equal sign with no spaces on -either side): - crnt_line = \"${crnt_line}\" - var_name = \"${var_name}\" -Continuing to next line in \"default_var_defns\"." - - fi - -done <<< "${default_var_defns}" -# -#----------------------------------------------------------------------- -# -# Construct the experiment's variable definitions file. Below, we first -# record the contents we want to place in this file in the variable -# var_defns_file_contents, and we then write the contents of this -# variable to the file. -# -#----------------------------------------------------------------------- -# -print_info_msg " -Generating the global experiment variable definitions file specified by -GLOBAL_VAR_DEFNS_FN: - GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" -Full path to this file is: - GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" -For more detailed information, set DEBUG to \"TRUE\" in the experiment -configuration file (\"${EXPT_CONFIG_FN}\")." - -var_defns_file_contents="\ -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 1: -# This section contains definitions of the various constants defined in -# the file ${CONSTANTS_FN}. -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# -${constant_defns} -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 2: -# This section contains (most of) the primary experiment variables, i.e. -# those variables that are defined in the default configuration file -# (${EXPT_DEFAULT_CONFIG_FN}) and that can be reset via the user-specified -# experiment configuration file (${EXPT_CONFIG_FN}). -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# -${var_defns}" -# -# Append derived/secondary variable definitions (as well as comments) to -# the contents of the variable definitions file. -# -ensmem_names_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ENSMEM_NAMES[@]}") -ensmem_names_str=$(printf "( %s${escbksl_nl_or_null})" "${ensmem_names_str}") - -fv3_nml_ensmem_fps_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}") -fv3_nml_ensmem_fps_str=$(printf "( %s${escbksl_nl_or_null})" "${fv3_nml_ensmem_fps_str}") - -var_defns_file_contents=${var_defns_file_contents}"\ -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 3: -# This section defines variables that have been derived from the primary -# set of experiment variables above (we refer to these as \"derived\" or -# \"secondary\" variables). -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Full path to workflow (re)launch script, its log file, and the line -# that gets added to the cron table to launch this script if the flag -# USE_CRON_TO_RELAUNCH is set to \"TRUE\". -# -#----------------------------------------------------------------------- -# -WFLOW_LAUNCH_SCRIPT_FP='${WFLOW_LAUNCH_SCRIPT_FP}' -WFLOW_LAUNCH_LOG_FP='${WFLOW_LAUNCH_LOG_FP}' -CRONTAB_LINE='${CRONTAB_LINE}' -# -#----------------------------------------------------------------------- -# -# Directories. -# -#----------------------------------------------------------------------- -# -SR_WX_APP_TOP_DIR='${SR_WX_APP_TOP_DIR}' -HOMErrfs='$HOMErrfs' -USHDIR='$USHDIR' -SCRIPTSDIR='$SCRIPTSDIR' -JOBSDIR='$JOBSDIR' -SORCDIR='$SORCDIR' -SRC_DIR='${SRC_DIR}' -PARMDIR='$PARMDIR' -MODULES_DIR='${MODULES_DIR}' -EXECDIR='$EXECDIR' -FIXam='$FIXam' -FIXclim='$FIXclim' -FIXLAM='$FIXLAM' -FIXgsm='$FIXgsm' -FIXaer='$FIXaer' -FIXlut='$FIXlut' -COMROOT='$COMROOT' -COMOUT_BASEDIR='${COMOUT_BASEDIR}' -TEMPLATE_DIR='${TEMPLATE_DIR}' -VX_CONFIG_DIR='${VX_CONFIG_DIR}' -METPLUS_CONF='${METPLUS_CONF}' -MET_CONFIG='${MET_CONFIG}' -UFS_WTHR_MDL_DIR='${UFS_WTHR_MDL_DIR}' -UFS_UTILS_DIR='${UFS_UTILS_DIR}' -SFC_CLIMO_INPUT_DIR='${SFC_CLIMO_INPUT_DIR}' -TOPO_DIR='${TOPO_DIR}' -UPP_DIR='${UPP_DIR}' - -EXPTDIR='$EXPTDIR' -LOGDIR='$LOGDIR' -CYCLE_BASEDIR='${CYCLE_BASEDIR}' -GRID_DIR='${GRID_DIR}' -OROG_DIR='${OROG_DIR}' -SFC_CLIMO_DIR='${SFC_CLIMO_DIR}' - -NDIGITS_ENSMEM_NAMES='${NDIGITS_ENSMEM_NAMES}' -ENSMEM_NAMES=${ensmem_names_str} -FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} -# -#----------------------------------------------------------------------- -# -# Files. -# -#----------------------------------------------------------------------- -# -GLOBAL_VAR_DEFNS_FP='${GLOBAL_VAR_DEFNS_FP}' - -DATA_TABLE_FN='${DATA_TABLE_FN}' -DIAG_TABLE_FN='${DIAG_TABLE_FN}' -FIELD_TABLE_FN='${FIELD_TABLE_FN}' -MODEL_CONFIG_FN='${MODEL_CONFIG_FN}' -NEMS_CONFIG_FN='${NEMS_CONFIG_FN}' - -DATA_TABLE_TMPL_FN='${DATA_TABLE_TMPL_FN}' -DIAG_TABLE_TMPL_FN='${DIAG_TABLE_TMPL_FN}' -FIELD_TABLE_TMPL_FN='${FIELD_TABLE_TMPL_FN}' -MODEL_CONFIG_TMPL_FN='${MODEL_CONFIG_TMPL_FN}' -NEMS_CONFIG_TMPL_FN='${NEMS_CONFIG_TMPL_FN}' - -DATA_TABLE_TMPL_FP='${DATA_TABLE_TMPL_FP}' -DIAG_TABLE_TMPL_FP='${DIAG_TABLE_TMPL_FP}' -FIELD_TABLE_TMPL_FP='${FIELD_TABLE_TMPL_FP}' -FV3_NML_BASE_SUITE_FP='${FV3_NML_BASE_SUITE_FP}' -FV3_NML_YAML_CONFIG_FP='${FV3_NML_YAML_CONFIG_FP}' -FV3_NML_BASE_ENS_FP='${FV3_NML_BASE_ENS_FP}' -MODEL_CONFIG_TMPL_FP='${MODEL_CONFIG_TMPL_FP}' -NEMS_CONFIG_TMPL_FP='${NEMS_CONFIG_TMPL_FP}' - -CCPP_PHYS_SUITE_FN='${CCPP_PHYS_SUITE_FN}' -CCPP_PHYS_SUITE_IN_CCPP_FP='${CCPP_PHYS_SUITE_IN_CCPP_FP}' -CCPP_PHYS_SUITE_FP='${CCPP_PHYS_SUITE_FP}' - -FIELD_DICT_FN='${FIELD_DICT_FN}' -FIELD_DICT_IN_UWM_FP='${FIELD_DICT_IN_UWM_FP}' -FIELD_DICT_FP='${FIELD_DICT_FP}' - -DATA_TABLE_FP='${DATA_TABLE_FP}' -FIELD_TABLE_FP='${FIELD_TABLE_FP}' -FV3_NML_FN='${FV3_NML_FN}' -FV3_NML_FP='${FV3_NML_FP}' -NEMS_CONFIG_FP='${NEMS_CONFIG_FP}' - -FV3_EXEC_FP='${FV3_EXEC_FP}' - -LOAD_MODULES_RUN_TASK_FP='${LOAD_MODULES_RUN_TASK_FP}' - -THOMPSON_MP_CLIMO_FN='${THOMPSON_MP_CLIMO_FN}' -THOMPSON_MP_CLIMO_FP='${THOMPSON_MP_CLIMO_FP}' -# -#----------------------------------------------------------------------- -# -# Flag for creating relative symlinks (as opposed to absolute ones). -# -#----------------------------------------------------------------------- -# -RELATIVE_LINK_FLAG='${RELATIVE_LINK_FLAG}' -# -#----------------------------------------------------------------------- -# -# Parameters that indicate whether or not various parameterizations are -# included in and called by the physics suite. -# -#----------------------------------------------------------------------- -# -SDF_USES_RUC_LSM='${SDF_USES_RUC_LSM}' -SDF_USES_THOMPSON_MP='${SDF_USES_THOMPSON_MP}' -# -#----------------------------------------------------------------------- -# -# Grid configuration parameters needed regardless of grid generation -# method used. -# -#----------------------------------------------------------------------- -# -GTYPE='$GTYPE' -TILE_RGNL='${TILE_RGNL}' - -LON_CTR='${LON_CTR}' -LAT_CTR='${LAT_CTR}' -NX='${NX}' -NY='${NY}' -NHW='${NHW}' -STRETCH_FAC='${STRETCH_FAC}' - -RES_IN_FIXLAM_FILENAMES='${RES_IN_FIXLAM_FILENAMES}' -# -# If running the make_grid task, CRES will be set to a null string during -# the grid generation step. It will later be set to an actual value after -# the make_grid task is complete. -# -CRES='$CRES' -" -# -#----------------------------------------------------------------------- -# -# Append to the variable definitions file the defintions of grid parameters -# that are specific to the grid generation method used. -# -#----------------------------------------------------------------------- -# -grid_vars_str="" -if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - - grid_vars_str="\ -# -#----------------------------------------------------------------------- -# -# Grid configuration parameters for a regional grid generated from a -# global parent cubed-sphere grid. This is the method originally -# suggested by GFDL since it allows GFDL's nested grid generator to be -# used to generate a regional grid. However, for large regional domains, -# it results in grids that have an unacceptably large range of cell sizes -# (i.e. ratio of maximum to minimum cell size is not sufficiently close -# to 1). -# -#----------------------------------------------------------------------- -# -ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' -IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' -JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' -JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' -" - -elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - - grid_vars_str="\ -# -#----------------------------------------------------------------------- -# -# Grid configuration parameters for a regional grid generated independently -# of a global parent grid. This method was developed by Jim Purser of -# EMC and results in very uniform grids (i.e. ratio of maximum to minimum -# cell size is very close to 1). -# -#----------------------------------------------------------------------- -# -DEL_ANGLE_X_SG='${DEL_ANGLE_X_SG}' -DEL_ANGLE_Y_SG='${DEL_ANGLE_Y_SG}' -NEG_NX_OF_DOM_WITH_WIDE_HALO='${NEG_NX_OF_DOM_WITH_WIDE_HALO}' -NEG_NY_OF_DOM_WITH_WIDE_HALO='${NEG_NY_OF_DOM_WITH_WIDE_HALO}' -PAZI='${PAZI}' -" - -fi -var_defns_file_contents="${var_defns_file_contents}${grid_vars_str}" -# -#----------------------------------------------------------------------- -# -# Continue appending variable definitions to the variable definitions -# file. -# -#----------------------------------------------------------------------- -# -lbc_spec_fcst_hrs_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${LBC_SPEC_FCST_HRS[@]}") -lbc_spec_fcst_hrs_str=$(printf "( %s${escbksl_nl_or_null})" "${lbc_spec_fcst_hrs_str}") - -all_cdates_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ALL_CDATES[@]}") -all_cdates_str=$(printf "( %s${escbksl_nl_or_null})" "${all_cdates_str}") - -var_defns_file_contents=${var_defns_file_contents}"\ -# -#----------------------------------------------------------------------- -# -# Flag in the \"${MODEL_CONFIG_FN}\" file for coupling the ocean model to -# the weather model. -# -#----------------------------------------------------------------------- -# -CPL='${CPL}' -# -#----------------------------------------------------------------------- -# -# Name of the ozone parameterization. The value this gets set to depends -# on the CCPP physics suite being used. -# -#----------------------------------------------------------------------- -# -OZONE_PARAM='${OZONE_PARAM}' -# -#----------------------------------------------------------------------- -# -# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system -# directory in which the workflow scripts will look for the files generated -# by the external model specified in EXTRN_MDL_NAME_ICS. These files will -# be used to generate the input initial condition and surface files for -# the FV3-LAM. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_ICS='${EXTRN_MDL_SYSBASEDIR_ICS}' -# -#----------------------------------------------------------------------- -# -# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system -# directory in which the workflow scripts will look for the files generated -# by the external model specified in EXTRN_MDL_NAME_LBCS. These files -# will be used to generate the input lateral boundary condition files for -# the FV3-LAM. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_LBCS='${EXTRN_MDL_SYSBASEDIR_LBCS}' -# -#----------------------------------------------------------------------- -# -# Shift back in time (in units of hours) of the starting time of the ex- -# ternal model specified in EXTRN_MDL_NAME_LBCS. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_LBCS_OFFSET_HRS='${EXTRN_MDL_LBCS_OFFSET_HRS}' -# -#----------------------------------------------------------------------- -# -# Boundary condition update times (in units of forecast hours). Note that -# LBC_SPEC_FCST_HRS is an array, even if it has only one element. -# -#----------------------------------------------------------------------- -# -LBC_SPEC_FCST_HRS=${lbc_spec_fcst_hrs_str} -# -#----------------------------------------------------------------------- -# -# The number of cycles for which to make forecasts and the list of -# starting dates/hours of these cycles. -# -#----------------------------------------------------------------------- -# -NUM_CYCLES='${NUM_CYCLES}' -ALL_CDATES=${all_cdates_str} -# -#----------------------------------------------------------------------- -# -# Parameters that determine whether FVCOM data will be used, and if so, -# their location. -# -# If USE_FVCOM is set to \"TRUE\", then FVCOM data (in the file FVCOM_FILE -# located in the directory FVCOM_DIR) will be used to update the surface -# boundary conditions during the initial conditions generation task -# (MAKE_ICS_TN). -# -#----------------------------------------------------------------------- -# -USE_FVCOM='${USE_FVCOM}' -FVCOM_DIR='${FVCOM_DIR}' -FVCOM_FILE='${FVCOM_FILE}' -# -#----------------------------------------------------------------------- -# -# Computational parameters. -# -#----------------------------------------------------------------------- -# -NCORES_PER_NODE='${NCORES_PER_NODE}' -PE_MEMBER01='${PE_MEMBER01}' -# -#----------------------------------------------------------------------- -# -# IF DO_SPP is set to "TRUE", N_VAR_SPP specifies the number of physics -# parameterizations that are perturbed with SPP. If DO_LSM_SPP is set to -# "TRUE", N_VAR_LNDP specifies the number of LSM parameters that are -# perturbed. LNDP_TYPE determines the way LSM perturbations are employed -# and FHCYC_LSM_SPP_OR_NOT sets FHCYC based on whether LSM perturbations -# are turned on or not. -# -#----------------------------------------------------------------------- -# -N_VAR_SPP='${N_VAR_SPP}' -N_VAR_LNDP='${N_VAR_LNDP}' -LNDP_TYPE='${LNDP_TYPE}' -LNDP_MODEL_TYPE='${LNDP_MODEL_TYPE}' -FHCYC_LSM_SPP_OR_NOT='${FHCYC_LSM_SPP_OR_NOT}' -" -# -# Done with constructing the contents of the variable definitions file, -# so now write the contents to file. -# -printf "%s\n" "${var_defns_file_contents}" >> ${GLOBAL_VAR_DEFNS_FP} - -print_info_msg "$VERBOSE" " -Done generating the global experiment variable definitions file." -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Function ${func_name}() in \"${scrfunc_fn}\" completed successfully!!! -========================================================================" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the start of this script/function. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - -} -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# -setup - diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh deleted file mode 100644 index 2755e5709f..0000000000 --- a/ush/valid_param_vals.sh +++ /dev/null @@ -1,59 +0,0 @@ -# -# Source file containing useful constants. -# -. ./constants.sh -# -# Define valid values for various global experiment/workflow variables. -# -valid_vals_RUN_ENVIR=("nco" "community") -valid_vals_MACHINE=("WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS" "NOAACLOUD" "SINGULARITY" "GAEA") -valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") -valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") -valid_vals_WORKFLOW_MANAGER=("rocoto" "none") -valid_vals_PREDEF_GRID_NAME=( \ -"RRFS_CONUS_25km" \ -"RRFS_CONUS_13km" \ -"RRFS_CONUS_3km" \ -"RRFS_CONUScompact_25km" \ -"RRFS_CONUScompact_13km" \ -"RRFS_CONUScompact_3km" \ -"RRFS_SUBCONUS_3km" \ -"RRFS_AK_13km" \ -"RRFS_AK_3km" \ -"CONUS_25km_GFDLgrid" \ -"CONUS_3km_GFDLgrid" \ -"EMC_AK" \ -"EMC_HI" \ -"EMC_PR" \ -"EMC_GU" \ -"GSL_HAFSV0.A_25km" \ -"GSL_HAFSV0.A_13km" \ -"GSL_HAFSV0.A_3km" \ -"GSD_HRRR_AK_50km" \ -"RRFS_NA_13km" \ -"RRFS_NA_3km" \ -"SUBCONUS_Ind_3km" \ -"WoFS_3km" \ -) -valid_vals_CCPP_PHYS_SUITE=( \ -"FV3_GFS_2017_gfdlmp" \ -"FV3_GFS_2017_gfdlmp_regional" \ -"FV3_GFS_v15p2" \ -"FV3_GFS_v15_thompson_mynn_lam3km" \ -"FV3_GFS_v16" \ -"FV3_RRFS_v1beta" \ -"FV3_HRRR" \ -) -valid_vals_GFDLgrid_NUM_CELLS=("48" "96" "192" "384" "768" "1152" "3072") -valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") -valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") -valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2" "netcdf") -valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2" "netcdf") -valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid") -valid_vals_PREEXISTING_DIR_METHOD=("delete" "rename" "quit") -valid_vals_GTYPE=("regional") -valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_latlon") -valid_vals_DOT_OR_USCORE=("." "_") -valid_vals_FVCOM_WCSTART=("warm" "WARM" "cold" "COLD") -valid_vals_COMPILER=("intel" "gnu") -valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 4c701f777c..64759b5a33 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -42,7 +42,7 @@ valid_vals_CCPP_PHYS_SUITE: [ "FV3_RRFS_v1beta", "FV3_HRRR" ] -valid_vals_GFDLgrid_RES: [48, 96, 192, 384, 768, 1152, 3072] +valid_vals_GFDLgrid_NUM_CELLS: [48, 96, 192, 384, 768, 1152, 3072] valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "RAP", "HRRR", "NAM"] valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "RAP", "HRRR", "NAM"] valid_vals_USE_USER_STAGED_EXTRN_FILES: [True, False] From 8ce26ba5b34094e97b175d3b8e3caf3e5e5ce3cd Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:07:34 -0400 Subject: [PATCH 192/203] Remove wcoss_dell_p3 from workflow (#810) * remove wcoss_dell_p3 * remove block for tide and gyre --- modulefiles/module-setup.csh.inc | 28 ------ modulefiles/module-setup.sh.inc | 29 ------ .../tasks/wcoss_dell_p3/get_extrn_ics.local | 6 -- .../tasks/wcoss_dell_p3/get_extrn_lbcs.local | 6 -- modulefiles/tasks/wcoss_dell_p3/get_obs.local | 6 -- .../tasks/wcoss_dell_p3/make_grid.local | 7 -- .../tasks/wcoss_dell_p3/make_ics.local | 7 -- .../tasks/wcoss_dell_p3/make_lbcs.local | 7 -- .../tasks/wcoss_dell_p3/run_fcst.local | 7 -- modulefiles/tasks/wcoss_dell_p3/run_vx.local | 3 - scripts/exregional_make_orog.sh | 3 +- ush/get_crontab_contents.py | 38 +++----- ush/launch_FV3LAM_wflow.sh | 1 - ush/machine/wcoss_dell_p3.sh | 89 ------------------- ush/templates/FV3LAM_wflow.xml | 23 ----- ush/valid_param_vals.yaml | 2 +- 16 files changed, 16 insertions(+), 246 deletions(-) delete mode 100644 modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/get_obs.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/make_grid.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/make_ics.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/make_lbcs.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/run_fcst.local delete mode 100644 modulefiles/tasks/wcoss_dell_p3/run_vx.local delete mode 100644 ush/machine/wcoss_dell_p3.sh diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc index 5ac893cab8..cb6cefb90f 100644 --- a/modulefiles/module-setup.csh.inc +++ b/modulefiles/module-setup.csh.inc @@ -13,34 +13,6 @@ else if ( { test -d /scratch3 } ) then source /apps/lmod/lmod/init/$__ms_shell endif module purge -else if ( { test -d /gpfs/hps -a -e /etc/SuSE-release } ) then - # We are on NOAA Luna or Surge - if ( ! { module help >& /dev/null } ) then - source /opt/modules/default/init/$__ms_shell - endif - module purge - module purge - unset _LMFILES_ - unset LOADEDMODULES - module use /opt/modulefiles - module use /opt/cray/ari/modulefiles - module use /opt/cray/craype/default/alt-modulefiles - module use /opt/cray/alt-modulefiles - module use /gpfs/hps/nco/ops/nwprod/modulefiles - module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles - module use /usrx/local/prod/modulefiles -else if ( { test -d /dcom -a -d /hwrf } ) then - # We are on NOAA Tide or Gyre - if ( ! { module help >& /dev/null } ) then - source /usrx/local/Modules/default/init/$__ms_shell - endif - module purge -else if ( { test -L /usrx && sh -c "readlink /usrx 2> /dev/null | grep dell" } ) then - # We are on WCOSS Mars or Venus - if ( ! { module help >& /dev/null } ) then - source /usrx/local/prod/lmod/lmod/init/$__ms_shell - endif - module purge else if ( { test -d /glade } ) then # We are on NCAR Yellowstone if ( ! { module help >& /dev/null } ) then diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index ec81c0187b..bf07ad9b13 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -28,35 +28,6 @@ elif [[ -d /scratch3 ]] ; then source /apps/lmod/lmod/init/$__ms_shell fi module purge -elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then - # We are on NOAA Luna or Surge - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /opt/modules/default/init/$__ms_shell - fi - module purge - module purge - # Workaround until module issues are fixed: - unset _LMFILES_ - unset LOADEDMODULES - module use /opt/modulefiles - module use /opt/cray/ari/modulefiles - module use /opt/cray/craype/default/alt-modulefiles - module use /opt/cray/alt-modulefiles - module use /gpfs/hps/nco/ops/nwprod/modulefiles - module use /gpfs/hps/nco/ops/nwprod/lib/modulefiles - module use /usrx/local/prod/modulefiles -elif [[ -d /dcom && -d /hwrf ]] ; then - # We are on NOAA Tide or Gyre - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /usrx/local/Modules/default/init/$__ms_shell - fi - module purge -elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then - # We are on NOAA Mars or Venus - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /usrx/local/prod/lmod/lmod/init/$__ms_shell - fi - module purge elif [[ -d /lustre && -d /ncrc ]] ; then # We are on GAEA. if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local deleted file mode 100644 index 9900015d1f..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local +++ /dev/null @@ -1,6 +0,0 @@ -#%Module##################################################### -## Module file for get_extrn_ics task. -############################################################# - -module purge -module load HPSS/5.0.2.5 diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local deleted file mode 100644 index 4a8b7431f9..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local +++ /dev/null @@ -1,6 +0,0 @@ -#%Module##################################################### -## Module file for get_extrn_lbcs task. -############################################################# - -module purge -module load HPSS/5.0.2.5 diff --git a/modulefiles/tasks/wcoss_dell_p3/get_obs.local b/modulefiles/tasks/wcoss_dell_p3/get_obs.local deleted file mode 100644 index 825bfe3c3a..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/get_obs.local +++ /dev/null @@ -1,6 +0,0 @@ -#%Module##################################################### -## Module file for get_obs_ccpa, get_obs_mrms, get_obs_ndas -############################################################# - -module purge -module load HPSS/5.0.2.5 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_grid.local b/modulefiles/tasks/wcoss_dell_p3/make_grid.local deleted file mode 100644 index 0e25550649..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/make_grid.local +++ /dev/null @@ -1,7 +0,0 @@ -#%Module - -module load lsf/10.1 -module load python/3.6.3 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_ics.local b/modulefiles/tasks/wcoss_dell_p3/make_ics.local deleted file mode 100644 index 0e25550649..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/make_ics.local +++ /dev/null @@ -1,7 +0,0 @@ -#%Module - -module load lsf/10.1 -module load python/3.6.3 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local deleted file mode 100644 index 0e25550649..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local +++ /dev/null @@ -1,7 +0,0 @@ -#%Module - -module load lsf/10.1 -module load python/3.6.3 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local deleted file mode 100644 index 5b62d935fd..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local +++ /dev/null @@ -1,7 +0,0 @@ -#%Module - -module unload python/2.7.14 -module load python/3.6.3 - -module use /usrx/local/nceplibs/dev/modulefiles -module load srw-app-python/1.0.0 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_vx.local b/modulefiles/tasks/wcoss_dell_p3/run_vx.local deleted file mode 100644 index 3806a6c4c2..0000000000 --- a/modulefiles/tasks/wcoss_dell_p3/run_vx.local +++ /dev/null @@ -1,3 +0,0 @@ -#%Module - -module unload netcdf diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 3904924433..e0a5b48935 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -68,8 +68,7 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -# Set OpenMP variables. The orog executable runs with OMP. On -# WCOSS (Cray), it is optimized for six threads, which is the default. +# Set OpenMP variables. The orog executable runs with OMP. # #----------------------------------------------------------------------- # diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 3f885ad978..50d0aea1f7 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -42,22 +42,18 @@ def get_crontab_contents(called_from_cron): IMPORTS = ["MACHINE", "USER"] import_vars(env_vars=IMPORTS) - if MACHINE == "WCOSS_DELL_P3": - __crontab_cmd__="" - (_,__crontab_contents__,_)=run_command(f'''cat "/u/{USER}/cron/mycrontab"''') - else: - __crontab_cmd__="crontab" - # - # On Cheyenne, simply typing "crontab" will launch the crontab command - # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized - # version of crontab that will work if called from scripts that are - # themselves being called as cron jobs. In that case, we must instead - # call the system version of crontab at /usr/bin/crontab. - # - if MACHINE == "CHEYENNE": - if called_from_cron: - __crontab_cmd__="/usr/bin/crontab" - (_,__crontab_contents__,_)=run_command(f'''{__crontab_cmd__} -l''') + __crontab_cmd__="crontab" + # + # On Cheyenne, simply typing "crontab" will launch the crontab command + # at "/glade/u/apps/ch/opt/usr/bin/crontab". This is a containerized + # version of crontab that will work if called from scripts that are + # themselves being called as cron jobs. In that case, we must instead + # call the system version of crontab at /usr/bin/crontab. + # + if MACHINE == "CHEYENNE": + if called_from_cron: + __crontab_cmd__="/usr/bin/crontab" + (_,__crontab_contents__,_)=run_command(f'''{__crontab_cmd__} -l''') return __crontab_cmd__, __crontab_contents__ @@ -108,10 +104,7 @@ def add_crontab_line(): NEWLINE_CHAR="\n" #add the crontab line - if MACHINE == "WCOSS_DELL_P3": - run_command(f'''printf "%b%s\n" '{NEWLINE_CHAR}' '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''') - else: - run_command(f'''printf "%s%b%s\n" '{crontab_contents}' '{NEWLINE_CHAR}' '{CRONTAB_LINE}' | {crontab_cmd}''') + run_command(f'''printf "%s%b%s\n" '{crontab_contents}' '{NEWLINE_CHAR}' '{CRONTAB_LINE}' | {crontab_cmd}''') def delete_crontab_line(called_from_cron): """ Delete crontab line after job is complete i.e. either SUCCESS/FAILURE @@ -137,10 +130,7 @@ def delete_crontab_line(called_from_cron): else: crontab_contents = crontab_contents.replace(CRONTAB_LINE,'') - if MACHINE == "WCOSS_DELL_P3": - run_command(f'''echo '{crontab_contents}' > "/u/{USER}/cron/mycrontab"''') - else: - run_command(f'''echo '{crontab_contents}' | {crontab_cmd}''') + run_command(f'''echo '{crontab_contents}' | {crontab_cmd}''') def parse_args(argv): """ Parse command line arguments for deleting crontab line. diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 17734b9477..de05d0e087 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -401,7 +401,6 @@ script for this experiment: MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \ python3 $USHDIR/get_crontab_contents.py --delete fi - fi # # Print the workflow completion message to the launch log file. diff --git a/ush/machine/wcoss_dell_p3.sh b/ush/machine/wcoss_dell_p3.sh deleted file mode 100644 index c4b6b038d7..0000000000 --- a/ush/machine/wcoss_dell_p3.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - case ${external_model} in - - "FV3GFS") - location='/gpfs/dell1/nco/ops/com/gfs/prod/gfs.${yyyymmdd}/${hh}/atmos' - ;; - "RAP") - location='/gpfs/hps/nco/ops/com/rap/prod' - ;; - "HRRR") - location='/gpfs/hps/nco/ops/com/hrrr/prod' - ;; - "NAM") - location='/gpfs/dell1/nco/ops/com/nam/prod' - ;; - - esac - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" "/usrx/local/prod/lmod/lmod/init/sh" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-24} -SCHED=${SCHED:-"lsf"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} -QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} -QUEUE_FCST=${QUEUE_FCST:-"dev"} - -# UFS SRW App specific paths -staged_data_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="mpirun" -RUN_CMD_UTILS="mpirun" -RUN_CMD_FCST='mpirun -l -np ${PE_MEMBER01}' -RUN_CMD_POST="mpirun" - -# MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.1.1"} -METPLUS_PATH=${METPLUS_PATH:-"/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.1.1"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"exec"} - -# Test Data Locations -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_COMIN="${staged_data_dir}/COMGFS" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" - -# Computational resources -WTIME_MAKE_OROG="01:00:00" - diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 8dff08780d..219798afb1 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -178,11 +178,7 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_GRID_TN;" "&JOBSDIR;/JREGIONAL_MAKE_GRID" - {%- if machine in ["WCOSS_DELL_P3"] %} - {{ nnodes_make_grid }}:ppn=1 - {%- else %} {{ nnodes_make_grid }}:ppn={{ ppn_make_grid }} - {%- endif %} {{ wtime_make_grid }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} @@ -204,11 +200,7 @@ MODULES_RUN_TASK_FP script. &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_OROG_TN;" "&JOBSDIR;/JREGIONAL_MAKE_OROG" - {%- if machine in ["WCOSS_DELL_P3"] %} - {{ nnodes_make_orog }}:ppn=1 - {%- else %} {{ nnodes_make_orog }}:ppn={{ ppn_make_orog }} - {%- endif %} {{ wtime_make_orog }} &NCORES_PER_NODE; {%- if machine in ["GAEA"] %} @@ -275,9 +267,6 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_ICS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_get_extrn_ics }}:ppn={{ ppn_get_extrn_ics }} {{ wtime_get_extrn_ics }} &NCORES_PER_NODE; @@ -304,9 +293,6 @@ MODULES_RUN_TASK_FP script. &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_LBCS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }} {{ wtime_get_extrn_lbcs }} &NCORES_PER_NODE; @@ -728,9 +714,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_CCPA" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_get_obs_ccpa }}:ppn={{ ppn_get_obs_ccpa }} {{ wtime_get_obs_ccpa }} &NCORES_PER_NODE; @@ -761,9 +744,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_MRMS" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_get_obs_mrms }}:ppn={{ ppn_get_obs_mrms }} {{ wtime_get_obs_mrms }} &NCORES_PER_NODE; @@ -795,9 +775,6 @@ the tag to be identical to the ones above for other output times. &RSRV_HPSS; &LOAD_MODULES_RUN_TASK_FP; "&GET_OBS;" "&JOBSDIR;/JREGIONAL_GET_OBS_NDAS" - {%- if machine in ["WCOSS_DELL_P3"] %} - 2048M-R affinity[core] - {%- endif %} {{ nnodes_get_obs_ndas }}:ppn={{ ppn_get_obs_ndas }} {{ wtime_get_obs_ndas }} &NCORES_PER_NODE; diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index 64759b5a33..4d6b40278a 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -4,7 +4,7 @@ valid_vals_RUN_ENVIR: ["nco", "community"] valid_vals_VERBOSE: [True, False] valid_vals_DEBUG: [True, False] -valid_vals_MACHINE: ["WCOSS_DELL_P3", "HERA", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] +valid_vals_MACHINE: ["HERA", "ORION", "JET", "ODIN", "CHEYENNE", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"] valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"] valid_vals_FCST_MODEL: ["ufs-weather-model", "fv3gfs_aqm"] valid_vals_WORKFLOW_MANAGER: ["rocoto", "none"] From 833ecb4adf09ef59371fa5754bca04cd4e0f2355 Mon Sep 17 00:00:00 2001 From: Michael Kavulich Date: Thu, 21 Jul 2022 11:48:42 -0600 Subject: [PATCH 193/203] Replace deprecated NCAR python environment with conda on Cheyenne (#812) --- modulefiles/tasks/cheyenne/conda_regional_workflow | 5 +++++ modulefiles/tasks/cheyenne/get_extrn_ics | 4 ---- modulefiles/tasks/cheyenne/get_extrn_ics.local | 4 ++++ modulefiles/tasks/cheyenne/get_extrn_lbcs | 4 ---- modulefiles/tasks/cheyenne/get_extrn_lbcs.local | 4 ++++ modulefiles/tasks/cheyenne/make_grid.local | 2 +- modulefiles/tasks/cheyenne/make_ics.local | 2 +- modulefiles/tasks/cheyenne/make_lbcs.local | 2 +- modulefiles/tasks/cheyenne/pylib_regional_workflow | 9 --------- modulefiles/tasks/cheyenne/run_fcst.local | 2 +- modulefiles/tasks/cheyenne/run_vx.local | 2 +- ush/Python/qsub_job.sh | 4 +++- ush/Python/qsub_job_diff.sh | 4 +++- ush/wrappers/qsub_job.sh | 3 ++- 14 files changed, 26 insertions(+), 25 deletions(-) create mode 100644 modulefiles/tasks/cheyenne/conda_regional_workflow delete mode 100644 modulefiles/tasks/cheyenne/get_extrn_ics create mode 100644 modulefiles/tasks/cheyenne/get_extrn_ics.local delete mode 100644 modulefiles/tasks/cheyenne/get_extrn_lbcs create mode 100644 modulefiles/tasks/cheyenne/get_extrn_lbcs.local delete mode 100644 modulefiles/tasks/cheyenne/pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/conda_regional_workflow b/modulefiles/tasks/cheyenne/conda_regional_workflow new file mode 100644 index 0000000000..e490af9091 --- /dev/null +++ b/modulefiles/tasks/cheyenne/conda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module unload python +module load conda +setenv SRW_ENV /glade/p/ral/jntp/UFS_SRW_app/conda/regional_workflow + diff --git a/modulefiles/tasks/cheyenne/get_extrn_ics b/modulefiles/tasks/cheyenne/get_extrn_ics deleted file mode 100644 index 9f7a4e4f73..0000000000 --- a/modulefiles/tasks/cheyenne/get_extrn_ics +++ /dev/null @@ -1,4 +0,0 @@ -#%Module - -module load pylib_regional_workflow - diff --git a/modulefiles/tasks/cheyenne/get_extrn_ics.local b/modulefiles/tasks/cheyenne/get_extrn_ics.local new file mode 100644 index 0000000000..7f0ed16e4f --- /dev/null +++ b/modulefiles/tasks/cheyenne/get_extrn_ics.local @@ -0,0 +1,4 @@ +#%Module + +module load conda_regional_workflow + diff --git a/modulefiles/tasks/cheyenne/get_extrn_lbcs b/modulefiles/tasks/cheyenne/get_extrn_lbcs deleted file mode 100644 index 9f7a4e4f73..0000000000 --- a/modulefiles/tasks/cheyenne/get_extrn_lbcs +++ /dev/null @@ -1,4 +0,0 @@ -#%Module - -module load pylib_regional_workflow - diff --git a/modulefiles/tasks/cheyenne/get_extrn_lbcs.local b/modulefiles/tasks/cheyenne/get_extrn_lbcs.local new file mode 100644 index 0000000000..7f0ed16e4f --- /dev/null +++ b/modulefiles/tasks/cheyenne/get_extrn_lbcs.local @@ -0,0 +1,4 @@ +#%Module + +module load conda_regional_workflow + diff --git a/modulefiles/tasks/cheyenne/make_grid.local b/modulefiles/tasks/cheyenne/make_grid.local index 2f92a39e57..e708b857cc 100644 --- a/modulefiles/tasks/cheyenne/make_grid.local +++ b/modulefiles/tasks/cheyenne/make_grid.local @@ -1,2 +1,2 @@ #%Module -module load pylib_regional_workflow +module load conda_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_ics.local b/modulefiles/tasks/cheyenne/make_ics.local index 2f92a39e57..e708b857cc 100644 --- a/modulefiles/tasks/cheyenne/make_ics.local +++ b/modulefiles/tasks/cheyenne/make_ics.local @@ -1,2 +1,2 @@ #%Module -module load pylib_regional_workflow +module load conda_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_lbcs.local b/modulefiles/tasks/cheyenne/make_lbcs.local index 2f92a39e57..e708b857cc 100644 --- a/modulefiles/tasks/cheyenne/make_lbcs.local +++ b/modulefiles/tasks/cheyenne/make_lbcs.local @@ -1,2 +1,2 @@ #%Module -module load pylib_regional_workflow +module load conda_regional_workflow diff --git a/modulefiles/tasks/cheyenne/pylib_regional_workflow b/modulefiles/tasks/cheyenne/pylib_regional_workflow deleted file mode 100644 index 4232b86593..0000000000 --- a/modulefiles/tasks/cheyenne/pylib_regional_workflow +++ /dev/null @@ -1,9 +0,0 @@ -#%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - diff --git a/modulefiles/tasks/cheyenne/run_fcst.local b/modulefiles/tasks/cheyenne/run_fcst.local index 2f92a39e57..e708b857cc 100644 --- a/modulefiles/tasks/cheyenne/run_fcst.local +++ b/modulefiles/tasks/cheyenne/run_fcst.local @@ -1,2 +1,2 @@ #%Module -module load pylib_regional_workflow +module load conda_regional_workflow diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local index 2f92a39e57..e708b857cc 100644 --- a/modulefiles/tasks/cheyenne/run_vx.local +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -1,2 +1,2 @@ #%Module -module load pylib_regional_workflow +module load conda_regional_workflow diff --git a/ush/Python/qsub_job.sh b/ush/Python/qsub_job.sh index 4640208767..df3ff20be3 100755 --- a/ush/Python/qsub_job.sh +++ b/ush/Python/qsub_job.sh @@ -19,7 +19,9 @@ source /etc/profile.d/modules.sh ############ # Python environment for Cheyenne ############ -ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics +module load ncarenv +module load conda/latest +conda activate /glade/p/ral/jntp/UFS_SRW_app/conda/python_graphics ############ # Path to shape files diff --git a/ush/Python/qsub_job_diff.sh b/ush/Python/qsub_job_diff.sh index c3f5e4fba6..d2b1425c6a 100755 --- a/ush/Python/qsub_job_diff.sh +++ b/ush/Python/qsub_job_diff.sh @@ -20,7 +20,9 @@ source /etc/profile.d/modules.sh ############ # Python environment for Cheyenne ############ -ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics +module load ncarenv +module load conda/latest +conda activate /glade/p/ral/jntp/UFS_SRW_app/conda/python_graphics ############ # Path to shape files diff --git a/ush/wrappers/qsub_job.sh b/ush/wrappers/qsub_job.sh index 15ec7eb140..8cd49aa57c 100755 --- a/ush/wrappers/qsub_job.sh +++ b/ush/wrappers/qsub_job.sh @@ -38,7 +38,8 @@ module load esmf/8.0.0 # #module use /glade/p/ral/jntp/MET/MET_releases/modulefiles #module load met/10.0.0 -#ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib/python_graphics +#module load conda/latest +#conda activate /glade/p/ral/jntp/UFS_SRW_app/conda/python_graphics #./run_pointvx.sh # Run grod-to-point deterministic vx #./run_gridvx.sh # Run grid-stat deterministic vx From be534f23627279f6a4de2189e84161b027f2e390 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Fri, 22 Jul 2022 13:40:58 -0400 Subject: [PATCH 194/203] Fix issue on get_extrn_lbcs when FCST_LEN_HRS>=40 with netcdf (#814) * activate b file on hpss for >40h * add a new we2e test for fcst_len_hrs>40 * reduce fcst time for we2e --- ...S_lbcs_FV3GFS_fmt_netcdf_2022060112_48h.sh | 26 +++++++++++++++++ ush/retrieve_data.py | 29 +++++++++++++++---- 2 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h.sh diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h.sh new file mode 100644 index 0000000000..c119306516 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h.sh @@ -0,0 +1,26 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOAA +# HPSS netcdf-formatted output files generated by the FV3GFS external +# model (FCST_LEN_HRS>=40). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v16" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="netcdf" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_ICS="netcdf" + +DATE_FIRST_CYCL="20220601" +DATE_LAST_CYCL="20220601" +CYCL_HRS=( "12" ) + +FCST_LEN_HRS="48" +LBC_SPEC_INTVL_HRS="12" diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index d68c082370..4267d6faaf 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -214,6 +214,8 @@ def find_archive_files(paths, file_names, cycle_date): zipped_archive_file_paths = zip(paths, file_names) + existing_archive={} + # Narrow down which HPSS files are available for this date for list_item, (archive_path, archive_file_names) in \ enumerate(zipped_archive_file_paths): @@ -226,9 +228,14 @@ def find_archive_files(paths, file_names, cycle_date): file_path = os.path.join(archive_path, archive_file_names[0]) file_path = fill_template(file_path, cycle_date) - existing_archive = hsi_single_file(file_path) + existing_archive[0] = hsi_single_file(file_path) + + if len(archive_file_names)>1: + file_path_b = os.path.join(archive_path, archive_file_names[1]) + file_path_b = fill_template(file_path_b, cycle_date) + existing_archive[1] = hsi_single_file(file_path_b) - if existing_archive: + if existing_archive[0]: logging.info(f'Found HPSS file: {file_path}') return existing_archive, list_item @@ -387,6 +394,7 @@ def hpss_requested_files(cla, file_names, store_specs): source_paths = [] for fcst_hr in cla.fcst_hrs: for file_name in file_names: + source_paths.append(fill_template( os.path.join(archive_internal_dir, file_name), cla.cycle_date, @@ -395,13 +403,13 @@ def hpss_requested_files(cla, file_names, store_specs): if store_specs.get('archive_format', 'tar') == 'zip': # Get the entire file from HPSS - existing_archive = hsi_single_file(existing_archive, mode='get') + existing_archive[0] = hsi_single_file(existing_archive[0], mode='get') # Grab only the necessary files from the archive - cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + cmd = f'unzip -o {os.path.basename(existing_archive[0])} {" ".join(source_paths)}' else: - cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + cmd = f'htar -xvf {existing_archive[0]} {" ".join(source_paths)}' logging.info(f'Running command \n {cmd}') subprocess.run(cmd, @@ -409,13 +417,22 @@ def hpss_requested_files(cla, file_names, store_specs): shell=True, ) + if len(existing_archive)>1: + cmd = f'htar -xvf {existing_archive[1]} {" ".join(source_paths)}' + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) + # Check that files exist and Remove any data transfer artifacts. unavailable = clean_up_output_dir( expected_subdir=archive_internal_dir, - local_archive=os.path.basename(existing_archive), + local_archive=os.path.basename(existing_archive[0]), output_path=output_path, source_paths=source_paths, ) + if not unavailable: return unavailable From 66e185fccfca67eaa5dbd9d89f49c15db898fb77 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:29:28 -0600 Subject: [PATCH 195/203] Fixes a problem with adding crontabline when the existing lines have single quotes. (#816) --- ush/get_crontab_contents.py | 3 +++ ush/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 50d0aea1f7..69bdad9faa 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -55,6 +55,9 @@ def get_crontab_contents(called_from_cron): __crontab_cmd__="/usr/bin/crontab" (_,__crontab_contents__,_)=run_command(f'''{__crontab_cmd__} -l''') + # replace single quotes (hopefully in comments) with double quotes + __crontab_contents__ = __crontab_contents__.replace("'", '"') + return __crontab_cmd__, __crontab_contents__ def add_crontab_line(): diff --git a/ush/setup.py b/ush/setup.py index 7a3cb032f0..6a8f35231d 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -329,7 +329,7 @@ def setup(): SRC_DIR = os.path.join(SR_WX_APP_TOP_DIR,"src") PARMDIR = os.path.join(HOMErrfs,"parm") MODULES_DIR = os.path.join(HOMErrfs,"modulefiles") - EXECDIR = os.path.join(SR_WX_APP_TOP_DIR,"bin") + EXECDIR = os.path.join(SR_WX_APP_TOP_DIR,EXEC_SUBDIR) TEMPLATE_DIR = os.path.join(USHDIR,"templates") VX_CONFIG_DIR = os.path.join(TEMPLATE_DIR,"parm") METPLUS_CONF = os.path.join(TEMPLATE_DIR,"parm","metplus") From 871983faf73c2181c1abf4cbe7215443bdb2349c Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Tue, 9 Aug 2022 15:54:56 -0600 Subject: [PATCH 196/203] Bugfix for ensemble+stochastic physics issue and new WE2E test case for stochastic physics. (#819) * Add path_to_defns argument to set_FV3nml_ens_stoch_seeds. * Keep hour in directory name of ensemble members. * Fix SPP/T flag * Fix indentation of error/info message containing settings * Bugfix for DO_SHUM/SKEB which came from shell workflow generation * Fix nam_spp/p/erts confusion. * Make create_diag_table info/error message more informative. * Minor fix in info message. * Add a WE2E test case for testing stochastic physics. * Modify stoch phys test case. --- scripts/exregional_run_fcst.sh | 2 +- .../config.community_ensemble_2mems_stoch.sh | 38 +++++++++++++++++++ ush/create_diag_table_file.py | 20 +++++++--- ush/create_model_configure_file.py | 16 ++++---- ush/generate_FV3LAM_wflow.py | 24 ++++++------ ush/python_utils/environment.py | 11 ++---- ush/set_FV3nml_ens_stoch_seeds.py | 32 +++++++++++----- ush/set_FV3nml_sfc_climo_filenames.py | 12 +++--- 8 files changed, 102 insertions(+), 53 deletions(-) create mode 100644 tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems_stoch.sh diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 18b05d5c3c..f3dcabeb9a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -434,7 +434,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . fi -if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] \ +if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] || \ [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then python3 $USHDIR/set_FV3nml_ens_stoch_seeds.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ diff --git a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems_stoch.sh b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems_stoch.sh new file mode 100644 index 0000000000..934de78bb7 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems_stoch.sh @@ -0,0 +1,38 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to run ensemble forecasts +# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set +# to "community") with the number of ensemble members (NUM_ENS_MEMBERS) +# set to "2" and all stochastic physics options turned on. +# The lack of leading zeros in this "2" should cause the +# ensemble members to be named "mem1" and "mem2" (instead of, for instance, +# "mem01" and "mem02"). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="SUBCONUS_Ind_3km" +CCPP_PHYS_SUITE="FV3_HRRR" + +EXTRN_MDL_NAME_ICS="HRRR" +EXTRN_MDL_NAME_LBCS="HRRR" +EXTRN_MDL_FILES_ICS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}00{fcst_hr:02d}00' ) +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20200810" +DATE_LAST_CYCL="20200810" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="3" +LBC_SPEC_INTVL_HRS="3" + +DO_ENSEMBLE="TRUE" +NUM_ENS_MEMBERS="2" +DO_SHUM="TRUE" +DO_SPPT="TRUE" +DO_SKEB="TRUE" +DO_SPP="TRUE" diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 2752d630cc..0ac892727f 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -51,16 +51,24 @@ def create_diag_table_file(run_dir): } settings_str = cfg_to_yaml_str(settings) + print_info_msg(dedent(f''' + The variable \"settings\" specifying values to be used in the \"{DIAG_TABLE_FN}\" + file has been set as follows:\n + settings =\n\n''') + settings_str,verbose=VERBOSE) + #call fill jinja try: fill_jinja_template(["-q", "-u", settings_str, "-t", DIAG_TABLE_TMPL_FP, "-o", diag_table_fp]) except: - print_err_msg_exit(f''' - !!!!!!!!!!!!!!!!! - - fill_jinja_template.py failed! - - !!!!!!!!!!!!!!!!!''') + print_err_msg_exit(dedent(f''' + Call to python script fill_jinja_template.py to create a \"{DIAG_TABLE_FN}\" + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template diag table file: + DIAG_TABLE_TMPL_FP = \"{DIAG_TABLE_TMPL_FP}\" + Full path to output diag table file: + diag_table_fp = \"{diag_table_fp}\" + Namelist settings specified on command line:\n + settings =\n\n''') + settings_str) return False return True diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 1fafe8a7be..0ae12cd069 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -169,9 +169,8 @@ def create_model_configure_file(cdate,run_dir,sub_hourly_post,dt_subhourly_post_ print_info_msg(dedent(f''' The variable \"settings\" specifying values to be used in the \"{MODEL_CONFIG_FN}\" - file has been set as follows: - #----------------------------------------------------------------------- - settings =\n''') + settings_str,verbose=VERBOSE) + file has been set as follows:\n + settings =\n\n''') + settings_str,verbose=VERBOSE) # #----------------------------------------------------------------------- # @@ -185,16 +184,15 @@ def create_model_configure_file(cdate,run_dir,sub_hourly_post,dt_subhourly_post_ try: fill_jinja_template(["-q", "-u", settings_str, "-t", MODEL_CONFIG_TMPL_FP, "-o", model_config_fp]) except: - print_err_msg_exit(f''' + print_err_msg_exit(dedent(f''' Call to python script fill_jinja_template.py to create a \"{MODEL_CONFIG_FN}\" file from a jinja2 template failed. Parameters passed to this script are: - Full path to template rocoto XML file: + Full path to template model config file: MODEL_CONFIG_TMPL_FP = \"{MODEL_CONFIG_TMPL_FP}\" - Full path to output rocoto XML file: + Full path to output model config file: model_config_fp = \"{model_config_fp}\" - Namelist settings specified on command line: - settings = - {settings_str}''') + Namelist settings specified on command line:\n + settings =\n\n''') + settings_str) return False return True diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index f21fe308aa..65a3d16824 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -353,8 +353,8 @@ def generate_FV3LAM_wflow(): # # Parameters that determine the set of cycles to run. # - 'date_first_cycl': date_to_str(DATE_FIRST_CYCL,True), - 'date_last_cycl': date_to_str(DATE_LAST_CYCL,True), + 'date_first_cycl': date_to_str(DATE_FIRST_CYCL,format="%Y%m%d"), + 'date_last_cycl': date_to_str(DATE_LAST_CYCL,format="%Y%m%d"), 'cdate_first_cycl': cdate_first_cycl, 'cycl_hrs': cycl_hrs_str, 'cycl_freq': f"{INCR_CYCL_FREQ:02d}:00:00", @@ -402,7 +402,7 @@ def generate_FV3LAM_wflow(): The variable \"settings\" specifying values of the rococo XML variables has been set as follows: #----------------------------------------------------------------------- - settings =\n''') + settings_str, verbose=VERBOSE) + settings =\n\n''') + settings_str, verbose=VERBOSE) # # Call the python script to generate the experiment's actual XML file @@ -411,7 +411,7 @@ def generate_FV3LAM_wflow(): try: fill_jinja_template(["-q", "-u", settings_str, "-t", template_xml_fp, "-o", WFLOW_XML_FP]) except: - print_err_msg_exit(f''' + print_err_msg_exit(dedent(f''' Call to python script fill_jinja_template.py to create a rocoto workflow XML file from a template file failed. Parameters passed to this script are: @@ -419,9 +419,8 @@ def generate_FV3LAM_wflow(): template_xml_fp = \"{template_xml_fp}\" Full path to output rocoto XML file: WFLOW_XML_FP = \"{WFLOW_XML_FP}\" - Namelist settings specified on command line: - settings = - {settings_str}''') + Namelist settings specified on command line:\n + settings =\n\n''') + settings_str) # #----------------------------------------------------------------------- # @@ -810,7 +809,7 @@ def generate_FV3LAM_wflow(): The variable \"settings\" specifying values of the weather model's namelist variables has been set as follows: - settings =\n''') + settings_str, verbose=VERBOSE) + settings =\n\n''') + settings_str, verbose=VERBOSE) # #----------------------------------------------------------------------- # @@ -828,7 +827,7 @@ def generate_FV3LAM_wflow(): set_namelist(["-q", "-n", FV3_NML_BASE_SUITE_FP, "-c", FV3_NML_YAML_CONFIG_FP, CCPP_PHYS_SUITE, "-u", settings_str, "-o", FV3_NML_FP]) except: - print_err_msg_exit(f''' + print_err_msg_exit(dedent(f''' Call to python script set_namelist.py to generate an FV3 namelist file failed. Parameters passed to this script are: Full path to base namelist file: @@ -839,9 +838,8 @@ def generate_FV3LAM_wflow(): CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" Full path to output namelist file: FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line: - settings = - {settings_str}''') + Namelist settings specified on command line:\n + settings =\n\n''') + settings_str) # # If not running the MAKE_GRID_TN task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -954,7 +952,7 @@ def generate_FV3LAM_wflow(): print(f"NOMADS_file_type= {NOMADS_file_type}") cd_vrfy(EXPTDIR) NOMADS_script = os.path.join(USHDIR, "NOMADS_get_extrn_mdl_files.h") - run_command(f'''{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,True)} \ + run_command(f'''{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ {CYCL_HRS} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}''') diff --git a/ush/python_utils/environment.py b/ush/python_utils/environment.py index 8123130557..2dc6bfbd7c 100644 --- a/ush/python_utils/environment.py +++ b/ush/python_utils/environment.py @@ -29,20 +29,17 @@ def str_to_date(s): v = None return v -def date_to_str(d,short=False): +def date_to_str(d, format="%Y%m%d%H%M"): """ Get string from python datetime object. By default it converts to YYYYMMDDHHMM format unless - told otherwise with `short` or HHMM=0 + told otherwise by passing a different format Args: d: datetime object Returns: - string in YYYYMMDDHHMM or YYYYMMDD format + string in YYYYMMDDHHMM or shorter version of it """ - if short or (d.hour == 0 and d.minute == 0): - v = d.strftime("%Y%m%d") - else: - v = d.strftime("%Y%m%d%H%M") + v = d.strftime(format) return v def str_to_type(s, just_get_me_the_string = False): diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index b56979374f..76f30b39b0 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -48,16 +48,16 @@ def set_FV3nml_ens_stoch_seeds(cdate): # ensmem_name=f"mem{ENSMEM_INDX}" - fv3_nml_ensmem_fp=os.path.join(CYCLE_BASEDIR, f"{date_to_str(cdate,True)}{os.sep}{ensmem_name}{os.sep}{FV3_NML_FN}") + fv3_nml_ensmem_fp=os.path.join(CYCLE_BASEDIR, f'{date_to_str(cdate,format="%Y%m%d%H")}{os.sep}{ensmem_name}{os.sep}{FV3_NML_FN}') ensmem_num=ENSMEM_INDX - cdate_i = int(cdate.strftime('%Y%m%d')) + cdate_i = int(cdate.strftime('%Y%m%d%H')) settings = {} nam_stochy_dict = {} - if DO_SPP: + if DO_SPPT: iseed_sppt=cdate_i*1000 + ensmem_num*10 + 1 nam_stochy_dict.update({ 'iseed_sppt': iseed_sppt @@ -83,11 +83,11 @@ def set_FV3nml_ens_stoch_seeds(cdate): for i in range(num_iseed_spp): iseed_spp[i]=cdate_i*1000 + ensmem_num*10 + ISEED_SPP[i] - settings['nam_spperts'] = { + settings['nam_sppperts'] = { 'iseed_spp': iseed_spp } else: - settings['nam_spperts'] = {} + settings['nam_sppperts'] = {} if DO_LSM_SPP: iseed_lsm_spp=cdate_i*1000 + ensmem_num*10 + 9 @@ -97,6 +97,13 @@ def set_FV3nml_ens_stoch_seeds(cdate): } settings_str = cfg_to_yaml_str(settings) + + print_info_msg(dedent(f''' + The variable \"settings\" specifying seeds in \"{FV3_NML_FP}\" + has been set as follows: + + settings =\n\n''') + settings_str,verbose=VERBOSE) + try: set_namelist(["-q", "-n", FV3_NML_FP, "-u", settings_str, "-o", fv3_nml_ensmem_fp]) except: @@ -108,9 +115,8 @@ def set_FV3nml_ens_stoch_seeds(cdate): FV3_NML_FP = \"{FV3_NML_FP}\" Full path to output namelist file: fv3_nml_ensmem_fp = \"{fv3_nml_ensmem_fp}\" - Namelist settings specified on command line (these have highest precedence): - settings = - {settings_str}''')) + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n''') + settings_str) def parse_args(argv): """ Parse command line arguments""" @@ -123,6 +129,11 @@ def parse_args(argv): required=True, help='Date.') + parser.add_argument('-p', '--path-to-defns', + dest='path_to_defns', + required=True, + help='Path to var_defns file.') + return parser.parse_args(argv) if __name__ == '__main__': @@ -144,16 +155,17 @@ def setUp(self): cp_vrfy(os.path.join(USHDIR,f'templates{os.sep}input.nml.FV3'), \ os.path.join(EXPTDIR,'input.nml')) for i in range(2): - mkdir_vrfy("-p", os.path.join(EXPTDIR,f"{date_to_str(self.cdate,True)}{os.sep}mem{i+1}")) + mkdir_vrfy("-p", os.path.join(EXPTDIR,f'{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem{i+1}')) set_env_var("USHDIR",USHDIR) set_env_var("CYCLE_BASEDIR",EXPTDIR) set_env_var("ENSMEM_INDX",2) set_env_var("FV3_NML_FN","input.nml") set_env_var("FV3_NML_FP",os.path.join(EXPTDIR,"input.nml")) - set_env_var("DO_SPP",True) + set_env_var("DO_SPPT",True) set_env_var("DO_SHUM",True) set_env_var("DO_SKEB",True) + set_env_var("DO_SPP",True) set_env_var("DO_LSM_SPP",True) ISEED_SPP = [ 4, 5, 6, 7, 8] set_env_var("ISEED_SPP",ISEED_SPP) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index c1a4d68c65..d351d33d04 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -66,12 +66,11 @@ def set_FV3nml_sfc_climo_filenames(): settings_str = cfg_to_yaml_str(settings) - print_info_msg(f''' + print_info_msg(dedent(f''' The variable \"settings\" specifying values of the namelist variables has been set as follows: - settings = - {settings_str}''', verbose=DEBUG) + settings =\n\n''') + settings_str, verbose=VERBOSE) # Rename the FV3 namelist and call set_namelist fv3_nml_base_fp = f'{FV3_NML_FP}.base' @@ -80,7 +79,7 @@ def set_FV3nml_sfc_climo_filenames(): try: set_namelist(["-q", "-n", fv3_nml_base_fp, "-u", settings_str, "-o", FV3_NML_FP]) except: - print_err_msg_exit(f''' + print_err_msg_exit(dedent(f''' Call to python script set_namelist.py to set the variables in the FV3 namelist file that specify the paths to the surface climatology files failed. Parameters passed to this script are: @@ -88,9 +87,8 @@ def set_FV3nml_sfc_climo_filenames(): fv3_nml_base_fp = \"{fv3_nml_base_fp}\" Full path to output namelist file: FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line (these have highest precedence): - settings = - {settings_str}''') + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n''') + settings_str) rm_vrfy(f'{fv3_nml_base_fp}') From fa0d226d93da9166bb013137d68519314c493a8e Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Fri, 12 Aug 2022 10:08:10 -0600 Subject: [PATCH 197/203] Adding data ingest for global ensembles. (#805) Adds functionality to ush/retrieve_data.py that supports ingesting global ensemble members in the ways that have been used for RRFS. It also adds GEFS and GDAS known data locations to ush/templates/data_locations.yml. This means that it will pull data from AWS and/or HPSS for: - GEFS ICS and LBCS grib2 a & b files (it does not merge them, as that is outside the scope of this tool's functionality). - GDAS ICS and LBCS netcdf files. For ICS, it needs atm and sfc files. For LBCS, it only needs atm files. The tool is set up to pull all members as a single job, or it can be configured to do a subset of members if the user prefers to pull subsets of members. Just like for other datasets, the ICS and LBCS should be handled in separate calls to the tool, and the tool only handles a single start time since that is consistent with the behavior of the workflow. Additional functionality now includes the ability to template the output directory to include the ensemble member so each member can be stored in its own separate directory when the workflow runs. I have added functional tests to pull data from AWS and HPSS, and have hooked the AWS tests up to GitHub Actions to run when the relevant files have changed. No user intervention is needed to have those tests run -- they should run automatically upon opening a PR. --- .github/workflows/python_func_tests.yaml | 33 ++ ush/retrieve_data.py | 621 +++++++++++++++-------- ush/templates/data_locations.yml | 71 ++- ush/test_retrieve_data.py | 319 ++++++++++++ 4 files changed, 841 insertions(+), 203 deletions(-) create mode 100644 .github/workflows/python_func_tests.yaml create mode 100644 ush/test_retrieve_data.py diff --git a/.github/workflows/python_func_tests.yaml b/.github/workflows/python_func_tests.yaml new file mode 100644 index 0000000000..50e2f82986 --- /dev/null +++ b/.github/workflows/python_func_tests.yaml @@ -0,0 +1,33 @@ +name: Python unittests +on: + push: + paths: + - ush/*retrieve_data.py + - ush/templates/data_locations.yml + pull_request: + +env: + CI: true + +jobs: + python_functests: + name: Python Functional Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Install dependencies + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install python3 python3-pip netcdf-bin + sudo pip3 install pyyaml jinja2 f90nml + sudo pip3 install numpy matplotlib basemap + + # Run python functional tests + - name: Run python functional tests + run: | + cd ush + python3 -m unittest -b test_retrieve_data.FunctionalTesting diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 4267d6faaf..1257048e42 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -33,7 +33,7 @@ import subprocess import sys from textwrap import dedent - +import time import yaml @@ -113,7 +113,7 @@ def download_file(url): # -c continue previous attempt # -T timeout seconds # -t number of tries - cmd = f'wget -c -T 30 -t 3 {url}' + cmd = f'wget -q -c -T 30 -t 3 {url}' logging.info(f'Running command: \n {cmd}') try: subprocess.run(cmd, @@ -129,15 +129,14 @@ def download_file(url): return True -def fhr_list(args): +def arg_list_to_range(args): ''' - Given an argparse list argument, return the sequence of forecast hours to - process. + Given an argparse list argument, return the sequence to process. - The length of the list will determine what forecast hours are returned: + The length of the list will determine what sequence items are returned: - Length = 1: A single fhr is to be processed + Length = 1: A single item is to be processed Length = 2: A sequence of start, stop with increment 1 Length = 3: A sequence of start, stop, increment Length > 3: List as is @@ -155,8 +154,7 @@ def fhr_list(args): return args -def fill_template(template_str, cycle_date, fcst_hr=0, - templates_only=False): +def fill_template(template_str, cycle_date, templates_only=False, **kwargs): ''' Fill in the provided template string with date time information, and return the resulting string. @@ -165,15 +163,26 @@ def fill_template(template_str, cycle_date, fcst_hr=0, template_str a string containing Python templates cycle_date a datetime object that will be used to fill in date and time information - fcst_hr an integer forecast hour. string formatting should - be included in the template_str templates_only boolean value. When True, this function will only return the templates available. - Rerturn: + Keyword Args: + ens_group a number associated with a bin where ensemble + members are stored in archive files + fcst_hr an integer forecast hour. string formatting should + be included in the template_str + mem a single ensemble member. should be a positive integer value + + Return: filled template string ''' + # Parse keyword args + ens_group = kwargs.get('ens_group') + fcst_hr = kwargs.get('fcst_hr', 0) + mem = kwargs.get('mem', '') + # ----- + cycle_hour = cycle_date.strftime('%H') # One strategy for binning data files at NCEP is to put them into 6 # cycle bins. The archive file names include the low and high end of the @@ -189,11 +198,13 @@ def fill_template(template_str, cycle_date, fcst_hr=0, format_values = dict( bin6=bin6, + ens_group=ens_group, fcst_hr=fcst_hr, dd=cycle_date.strftime('%d'), hh=cycle_hour, hh_even=hh_even, jjj=cycle_date.strftime('%j'), + mem=mem, mm=cycle_date.strftime('%m'), yy=cycle_date.strftime('%y'), yyyy=cycle_date.strftime('%Y'), @@ -205,43 +216,86 @@ def fill_template(template_str, cycle_date, fcst_hr=0, return f'{",".join((format_values.keys()))}' return template_str.format(**format_values) -def find_archive_files(paths, file_names, cycle_date): +def create_target_path(target_path): + + ''' + Append target path and create directory for ensemble members + ''' + if not os.path.exists(target_path): + os.makedirs(target_path) + return target_path + +def find_archive_files(paths, file_names, cycle_date, ens_group): ''' Given an equal-length set of archive paths and archive file names, and a cycle date, check HPSS via hsi to make sure at least - one set exists. Return the path of the existing archive, along with + one set exists. Return a dict of the paths of the existing archive, along with the item in set of paths that was found.''' zipped_archive_file_paths = zip(paths, file_names) - existing_archive={} # Narrow down which HPSS files are available for this date for list_item, (archive_path, archive_file_names) in \ enumerate(zipped_archive_file_paths): + existing_archives = {} if not isinstance(archive_file_names, list): archive_file_names = [archive_file_names] + for n_fp, archive_file_name in enumerate(archive_file_names): # Only test the first item in the list, it will tell us if this # set exists at this date. - file_path = os.path.join(archive_path, archive_file_names[0]) - file_path = fill_template(file_path, cycle_date) + file_path = os.path.join(archive_path, archive_file_name) + file_path = fill_template(file_path, cycle_date, ens_group=ens_group) + file_path = hsi_single_file(file_path) - existing_archive[0] = hsi_single_file(file_path) + if file_path: + existing_archives[n_fp] = file_path - if len(archive_file_names)>1: - file_path_b = os.path.join(archive_path, archive_file_names[1]) - file_path_b = fill_template(file_path_b, cycle_date) - existing_archive[1] = hsi_single_file(file_path_b) - - if existing_archive[0]: - logging.info(f'Found HPSS file: {file_path}') - return existing_archive, list_item + if existing_archives: + for existing_archive in existing_archives.values(): + logging.info(f'Found HPSS file: {existing_archive}') + return existing_archives, list_item return '', 0 -def get_requested_files(cla, file_templates, input_loc, method='disk'): +def get_file_templates(cla, known_data_info, data_store, use_cla_tmpl=False): + + ''' Returns the file templates requested by user input, either from + the command line, or from the known data information dict. + + Arguments: + + cla command line arguments Namespace object + known_data_info dict from data_locations yaml file + data_store string corresponding to a key in the + known_data_info dict + use_cla_tmpl boolean on whether to check cla for templates + + Returns: + file_templates a list of file templates + ''' + + file_templates = known_data_info.get(data_store, {}).get('file_names') + if use_cla_tmpl: + file_templates = cla.file_templates if cla.file_templates else \ + file_templates + + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + if not file_templates: + msg = ('No file naming convention found. They must be provided \ + either on the command line or on in a config file.') + raise argparse.ArgumentTypeError(msg) + return file_templates + +def get_requested_files(cla, file_templates, input_locs, method='disk', + **kwargs): + + # pylint: disable=too-many-locals ''' This function copies files from disk locations or downloads files from a url, depending on the option specified for @@ -254,51 +308,97 @@ def get_requested_files(cla, file_templates, input_loc, method='disk'): cla Namespace object containing command line arguments file_templates a list of file templates - input_loc A string containing a single data location, either a url - or disk path. + input_locs A string containing a single data location, either a url + or disk path, or a list of paths/urls. method Choice of disk or download to indicate protocol for retrieval - Returns - unavailable a dict whose keys are "method" and whose values are a - list of files unretrievable + Keyword args: + members a list integers corresponding to the ensemble members + check_all boolean flag that indicates all urls should be + checked for all files + + Returns: + unavailable a list of locations/files that were unretrievable ''' - unavailable = {} + members = kwargs.get('members', '') + members = members if isinstance(members, list) else [members] + + + + check_all = kwargs.get('check_all', False) logging.info(f'Getting files named like {file_templates}') + # Make sure we're dealing with lists for input locations and file + # templates. Makes it easier to loop and zip. file_templates = file_templates if isinstance(file_templates, list) else \ - [file_templates] - target_path = fill_template(cla.output_path, - cla.cycle_date) + [file_templates] + + input_locs = input_locs if isinstance(input_locs, list) else \ + [input_locs] + + orig_path = os.path.dirname(__file__) + unavailable = [] + + locs_files = pair_locs_with_files(input_locs, file_templates, check_all) + for mem in members: + target_path = fill_template(cla.output_path, + cla.cycle_date, + mem=mem) + target_path = create_target_path(target_path) + + logging.info(f'Retrieved files will be placed here: \n {target_path}') + os.chdir(target_path) + + for fcst_hr in cla.fcst_hrs: + logging.debug(f'Looking for fhr = {fcst_hr}') + for loc, templates in locs_files: + + templates = templates if isinstance(templates, list) \ + else [templates] + + + logging.debug(f'Looking for files like {templates}') + logging.debug(f'They should be here: {loc}') + + template_loc = loc + for tmpl_num, template in enumerate(templates): + if isinstance(loc, list) and len(loc) == len(templates): + template_loc = loc[tmpl_num] + input_loc = os.path.join(template_loc, template) + input_loc = fill_template(input_loc, + cla.cycle_date, + fcst_hr=fcst_hr, + mem=mem, + ) + logging.debug(f'Full file path: {input_loc}') + + if method == 'disk': + retrieved = copy_file(input_loc, target_path) + + if method == 'download': + retrieved = download_file(input_loc) + # Wait a bit before trying the next download. + # Seems to reduce the occurrence of timeouts + # when downloading from AWS + time.sleep(15) + + logging.debug(f'Retrieved status: {retrieved}') + if not retrieved: + unavailable.append(input_loc) + # Go on to the next location if the first file + # isn't found here. + break + + # If retrieved, reset unavailable + unavailable = [] + if not unavailable: + # Start on the next fcst hour if all files were + # found from a loc/template combo + break - logging.info(f'Retrieved files will be placed here: \n {target_path}') - orig_path = os.getcwd() - os.chdir(target_path) - unavailable = {} - for fcst_hr in cla.fcst_hrs: - for file_template in file_templates: - loc = os.path.join(input_loc, file_template) - logging.debug(f'Full file path: {loc}') - loc = fill_template(loc, cla.cycle_date, fcst_hr) - - if method == 'disk': - retrieved = copy_file(loc, target_path) - - if method == 'download': - retrieved = download_file(loc) - - if not retrieved: - - if unavailable.get(method) is None: - unavailable[method] = [] - unavailable[method].append(target_path) - os.chdir(orig_path) - # Returning here assumes that if the first file - # isn't found, none of the others will be. Don't - # waste time timing out on every requested file. - return unavailable os.chdir(orig_path) return unavailable @@ -327,12 +427,15 @@ def hsi_single_file(file_path, mode='ls'): return file_path -def hpss_requested_files(cla, file_names, store_specs): +def hpss_requested_files(cla, file_names, store_specs, members=-1, + ens_group=-1): + + # pylint: disable=too-many-locals - ''' This function interacts with the "hpss" protocol in a - provided data store specs file to download a set of files requested - by the user. Depending on the type of archive file (zip or tar), it - will either pull the entire file and unzip it, or attempt to pull + ''' This function interacts with the "hpss" protocol in a provided + data store specs file to download a set of files requested by the + user. Depending on the type of archive file (zip or tar), it will + either pull the entire file and unzip it, or attempt to pull individual files from a tar file. It cleans up local disk after files are deemed available to remove @@ -341,6 +444,7 @@ def hpss_requested_files(cla, file_names, store_specs): This function exepcts that the output directory exists and is writable. ''' + members = [-1] if members == -1 else members archive_paths = store_specs['archive_path'] archive_paths = archive_paths if isinstance(archive_paths, list) \ @@ -355,17 +459,20 @@ def hpss_requested_files(cla, file_names, store_specs): archive_file_names = archive_file_names[cla.anl_or_fcst] unavailable = {} - existing_archive = None + existing_archives = {} logging.debug(f'Will try to look for: '\ f' {list(zip(archive_paths, archive_file_names))}') - existing_archive, which_archive = find_archive_files(archive_paths, + existing_archives, which_archive = find_archive_files(archive_paths, archive_file_names, cla.cycle_date, + ens_group=ens_group, ) - if not existing_archive: + logging.debug(f'Found existing archives: {existing_archives}') + + if not existing_archives: logging.warning('No archive files were found!') unavailable['archive'] = list(zip(archive_paths, archive_file_names)) return unavailable @@ -376,69 +483,94 @@ def hpss_requested_files(cla, file_names, store_specs): if isinstance(archive_internal_dirs, dict): archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, ['']) + # which_archive matters for choosing the correct file names within, # but we can safely just try all options for the # archive_internal_dir logging.debug(f'Checking archive number {which_archive} in list.') for archive_internal_dir_tmpl in archive_internal_dirs: - archive_internal_dir = fill_template(archive_internal_dir_tmpl, - cla.cycle_date) - - output_path = fill_template(cla.output_path, cla.cycle_date) - logging.info(f'Will place files in {os.path.abspath(output_path)}') - orig_path = os.getcwd() - os.chdir(output_path) - logging.debug(f'CWD: {os.getcwd()}') - - source_paths = [] - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - - source_paths.append(fill_template( - os.path.join(archive_internal_dir, file_name), - cla.cycle_date, - fcst_hr, - )) - - if store_specs.get('archive_format', 'tar') == 'zip': - # Get the entire file from HPSS - existing_archive[0] = hsi_single_file(existing_archive[0], mode='get') - - # Grab only the necessary files from the archive - cmd = f'unzip -o {os.path.basename(existing_archive[0])} {" ".join(source_paths)}' - - else: - cmd = f'htar -xvf {existing_archive[0]} {" ".join(source_paths)}' - - logging.info(f'Running command \n {cmd}') - subprocess.run(cmd, - check=True, - shell=True, - ) - - if len(existing_archive)>1: - cmd = f'htar -xvf {existing_archive[1]} {" ".join(source_paths)}' - logging.info(f'Running command \n {cmd}') - subprocess.run(cmd, - check=True, - shell=True, - ) - - # Check that files exist and Remove any data transfer artifacts. - unavailable = clean_up_output_dir( - expected_subdir=archive_internal_dir, - local_archive=os.path.basename(existing_archive[0]), - output_path=output_path, - source_paths=source_paths, - ) - - if not unavailable: - return unavailable + for mem in members: + archive_internal_dir = fill_template( + archive_internal_dir_tmpl, + cla.cycle_date, + mem=mem, + ) + + output_path = fill_template(cla.output_path, + cla.cycle_date, mem=mem) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + orig_path = os.path.dirname(__file__) + logging.debug(f'CWD: {os.getcwd()}') + os.chdir(orig_path) + + if mem != -1: + archive_internal_dir = fill_template(archive_internal_dir_tmpl, + cla.cycle_date, + mem=mem, + ) + output_path = create_target_path(output_path) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + + os.chdir(output_path) + + source_paths = [] + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + source_paths.append(fill_template( + os.path.join(archive_internal_dir, file_name), + cla.cycle_date, + fcst_hr=fcst_hr, + mem=mem, + ens_group=ens_group, + )) + + expected = set(source_paths) + unavailable = {} + for existing_archive in existing_archives.values(): + if store_specs.get('archive_format', 'tar') == 'zip': + + # Get the entire file from HPSS + existing_archive = hsi_single_file(existing_archive, mode='get') + + # Grab only the necessary files from the archive + cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + + else: + cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) + + # Check that files exist and Remove any data transfer artifacts. + # Returns {'hpss': []}, turn that into a new dict of + # sets. + unavailable[existing_archive] = set(clean_up_output_dir( + expected_subdir=archive_internal_dir, + local_archive=os.path.basename(existing_archive), + output_path=output_path, + source_paths=source_paths, + ).get('hpss', [])) + + # Once we go through all the archives, the union of all + # "unavailable" files should equal the "expected" list of + # files since clean_up_output_dir only reports on those that + # are missing from one of the files attempted. If any + # additional files are reported as unavailable, then + # something has gone wrong. + unavailable = set.union(*unavailable.values()) + + + # Report only the files that are truly unavailable + if not expected == unavailable: + return unavailable - expected os.chdir(orig_path) - return unavailable + return {} def load_str(arg): @@ -448,8 +580,8 @@ def load_str(arg): def config_exists(arg): ''' - Check to ensure that the provided config file exists. If it does, load it - with YAML's safe loader and return the resulting dict. + Check to ensure that the provided config file exists. If it does, + load it with YAML's safe loader and return the resulting dict. ''' # Check for existence of file @@ -461,6 +593,52 @@ def config_exists(arg): cfg = yaml.load(config_path, Loader=yaml.SafeLoader) return cfg +def pair_locs_with_files(input_locs, file_templates, check_all): + + ''' + Given a list of input locations and files, return an iterable that + contains the multiple locations and file templates for files that + should be searched in those locations. + + check_all indicates that all locations should be paired with all + avaiable file templates. + + The different possibilities: + 1. Get one or more files from a single path/url + 2. Get multiple files from multiple corresponding + paths/urls + 3. Check all paths for all file templates until files are + found + + The default will be to handle #1 and #2. #3 will be + indicated by a flag in the yaml: "check_all: True" + + ''' + + if not check_all: + + # Make sure the length of both input_locs and + # file_templates is consistent + + # Case 2 above + if len(file_templates) == len(input_locs): + locs_files = list(zip(input_locs, file_templates)) + + # Case 1 above + elif len(file_templates) > len(input_locs) and \ + len(input_locs) == 1: + + locs_files = list(zip(input_locs, [file_templates])) + else: + msg = "Please check your input locations and templates." + raise KeyError(msg) + else: + # Case 3 above + locs_files = [(loc, file_templates) for loc in input_locs] + + return locs_files + + def path_exists(arg): ''' Check whether the supplied path exists and is writeable ''' @@ -475,6 +653,7 @@ def path_exists(arg): return arg + def setup_logging(debug=False): ''' Calls initialization functions for logging package, and sets the @@ -492,13 +671,13 @@ def setup_logging(debug=False): def write_summary_file(cla, data_store, file_templates): - ''' Given the command line arguments and the data store from which the data - was retrieved, write a bash summary file that is needed by the workflow - elements downstream. ''' + ''' Given the command line arguments and the data store from which + the data was retrieved, write a bash summary file that is needed by + the workflow elements downstream. ''' files = [] for tmpl in file_templates: - files.extend([fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs]) + files.extend([fill_template(tmpl, cla.cycle_date, fcst_hr=fh) for fh in cla.fcst_hrs]) summary_fp = os.path.join(cla.output_path, cla.summary_file) logging.info(f'Writing a summary file to {summary_fp}') @@ -517,49 +696,85 @@ def write_summary_file(cla, data_store, file_templates): def to_datetime(arg): ''' Return a datetime object give a string like YYYYMMDDHH. ''' - return dt.datetime.strptime(arg, '%Y%m%d%H') + def to_lower(arg): ''' Return a string provided by arg into all lower case. ''' return arg.lower() -def main(cla): + +def main(argv): + # pylint: disable=too-many-branches, too-many-statements ''' Uses known location information to try the known locations and file paths in priority order. ''' - data_stores = cla.data_stores + cla = parse_args(argv) + cla.fcst_hrs = arg_list_to_range(cla.fcst_hrs) + + if cla.members: + cla.members = arg_list_to_range(cla.members) + + setup_logging(cla.debug) + print("Running script retrieve_data.py with args:\n", + f"{('-' * 80)}\n{('-' * 80)}") + for name, val in cla.__dict__.items(): + if name not in ['config']: + print(f"{name:>15s}: {val}") + print(f"{('-' * 80)}\n{('-' * 80)}") + + if 'disk' in cla.data_stores: + # Make sure a path was provided. + if not cla.input_file_path: + raise argparse.ArgumentTypeError( + ('You must provide an input_file_path when choosing ' \ + ' disk as a data store!')) + + if 'hpss' in cla.data_stores: + # Make sure hpss module is loaded + try: + subprocess.run('which hsi', + check=True, + shell=True, + ) + except subprocess.CalledProcessError: + logging.error('You requested the hpss data store, but ' \ + 'the HPSS module isn\'t loaded. This data store ' \ + 'is only available on NOAA compute platforms.') + sys.exit(1) + + known_data_info = cla.config.get(cla.external_model, {}) if not known_data_info: msg = dedent(f'''No data stores have been defined for - {cla.external_model}!''') + {cla.external_model}! Only checking provided disk + location''') if cla.input_file_path is None: - data_stores = ['disk'] + cla.data_stores = ['disk'] raise KeyError(msg) - logging.info(msg + ' Only checking provided disk location.') + logging.info(msg) unavailable = {} - for data_store in data_stores: + for data_store in cla.data_stores: logging.info(f'Checking {data_store} for {cla.external_model}') store_specs = known_data_info.get(data_store, {}) if data_store == 'disk': - file_templates = cla.file_templates if cla.file_templates else \ - known_data_info.get('hpss', {}).get('file_names') - if isinstance(file_templates, dict): - if cla.file_type is not None: - file_templates = file_templates[cla.file_type] - file_templates = file_templates[cla.anl_or_fcst] + file_templates = get_file_templates( + cla, + known_data_info, + data_store='hpss', + use_cla_tmpl=True, + ) + logging.debug(f'User supplied file names are: {file_templates}') - if not file_templates: - msg = ('No file naming convention found. They must be provided \ - either on the command line or on in a config file.') - raise argparse.ArgumentTypeError(msg) unavailable = get_requested_files(cla, + check_all=known_data_info.get('check_all', + False), file_templates=file_templates, - input_loc=cla.input_file_path, + input_locs=cla.input_file_path, method='disk', ) @@ -569,24 +784,32 @@ def main(cla): else: - file_templates = store_specs.get('file_names') - if isinstance(file_templates, dict): - if cla.file_type is not None: - file_templates = file_templates[cla.file_type] - file_templates = file_templates[cla.anl_or_fcst] - if not file_templates: - msg = ('No file name naming convention found. They must be provided \ - either on the command line or on in a config file.') - raise argparse.ArgumentTypeError(msg) + file_templates = get_file_templates( + cla, + known_data_info, + data_store=data_store, + ) if store_specs.get('protocol') == 'download': - unavailable = get_requested_files(cla, - file_templates=file_templates, - input_loc=store_specs['url'], - method='download', - ) + unavailable = get_requested_files( + cla, + check_all=known_data_info.get('check_all', False), + file_templates=file_templates, + input_locs=store_specs['url'], + method='download', + members=cla.members, + ) + if store_specs.get('protocol') == 'htar': - unavailable = hpss_requested_files(cla, file_templates, store_specs) + ens_groups = get_ens_groups(cla.members) + for ens_group, members in ens_groups.items(): + unavailable = hpss_requested_files( + cla, + file_templates, + store_specs, + members=members, + ens_group=ens_group, + ) if not unavailable: # All files are found. Stop looking! @@ -595,13 +818,33 @@ def main(cla): write_summary_file(cla, data_store, file_templates) break + logging.debug(f'Some unavailable files: {unavailable}') logging.warning(f'Requested files are unavailable from {data_store}') if unavailable: logging.error('Could not find any of the requested files.') sys.exit(1) -def parse_args(): +def get_ens_groups(members): + + ''' Given a list of ensemble members, return a dict with keys for + the ensemble group, and values are lists of ensemble members + requested in that group. ''' + + + if members is None: + return {-1: [-1]} + + ens_groups = {} + for mem in members: + ens_group = (mem - 1) // 10 + 1 + if ens_groups.get(ens_group) is None: + ens_groups[ens_group] = [mem] + else: + ens_groups[ens_group].append(mem) + return ens_groups + +def parse_args(argv): ''' Function maintains the arguments accepted by this script. Please see @@ -650,7 +893,7 @@ def parse_args(): ) parser.add_argument( '--external_model', - choices=('FV3GFS', 'GSMGFS', 'HRRR', 'NAM', 'RAP', 'RAPx', + choices=('FV3GFS','GDAS', 'GEFS', 'GSMGFS', 'HRRR', 'NAM', 'RAP', 'RAPx', 'HRRRx'), help='External model label. This input is case-sensitive', required=True, @@ -698,45 +941,23 @@ def parse_args(): --file_templates flag, or the default naming convention will be \ taken from the --config file.', ) + parser.add_argument( + '--members', + help='A list describing ensemble members. If one argument, \ + one member will be processed. If 2 or 3 arguments, a sequence \ + of members [start, stop, [increment]] will be \ + processed. If more than 3 arguments, the list is processed \ + as-is.', + nargs='*', + type=int, + ) parser.add_argument( '--summary_file', help='Name of the summary file to be written to the output \ directory', ) - return parser.parse_args() - -if __name__ == '__main__': - - CLA = parse_args() - CLA.output_path = path_exists(CLA.output_path) - CLA.fcst_hrs = fhr_list(CLA.fcst_hrs) + return parser.parse_args(argv) - setup_logging(CLA.debug) - print(f"Running script retrieve_data.py with args:\n", - f"{('-' * 80)}\n{('-' * 80)}") - for name, val in CLA.__dict__.items(): - if name not in ['config']: - print(f"{name:>15s}: {val}") - print(f"{('-' * 80)}\n{('-' * 80)}") - - if 'disk' in CLA.data_stores: - # Make sure a path was provided. - if not CLA.input_file_path: - raise argparse.ArgumentTypeError( - ('You must provide an input_file_path when choosing ' \ - ' disk as a data store!')) - - if 'hpss' in CLA.data_stores: - # Make sure hpss module is loaded - try: - output = subprocess.run('which hsi', - check=True, - shell=True, - ) - except subprocess.CalledProcessError: - logging.error('You requested the hpss data store, but ' \ - 'the HPSS module isn\'t loaded. This data store ' \ - 'is only available on NOAA compute platforms.') - - main(CLA) +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index ef5d49c8e7..067708f99f 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -24,9 +24,9 @@ # for that type of file. Both are required for external models ICS and # LBCS. Each file name may (and likely should) include templates. # -# If the SRW App accepts different file formats, -# as it does for FV3GFS, an intermediary level indicating the value -# associate with a FV3GFS_FILE_FMT_* variable. +# If the SRW App accepts different file formats, as it does for +# FV3GFS, an intermediary level indicating the value associate with a +# FV3GFS_FILE_FMT_* variable. # # 3rd level optional: # @@ -108,6 +108,68 @@ FV3GFS: file_names: <<: *gfs_file_names +GDAS: + hpss: + protocol: htar + archive_path: + - /NCEPPROD/5year/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/5year/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + - ./enkfgdas.{yyyymmdd}/{hh}/atmos/mem{mem:03d} + archive_file_names: + nemsio: + anl: + - com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + - gpfs_dell1_nco_ops_com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + fcst: + - com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + - gpfs_dell1_nco_ops_com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + netcdf: + anl: + - com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + - gpfs_dell1_nco_ops_com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + fcst: + - com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + - gpfs_dell1_nco_ops_com_gfs_prod_enkfgdas.{yyyymmdd}_{hh}.enkfgdas_grp{ens_group}.tar + file_names: + nemsio: + anl: + - gdas.t{hh}z.atmf{fcst_hr:03d}.nemsio + fcst: + - gdas.t{hh}.sfcf{fcst_hr:03d}.nemsio + netcdf: + anl: + - gdas.t{hh}z.atmf{fcst_hr:03d}.nc + - gdas.t{hh}z.sfcf{fcst_hr:03d}.nc + fcst: + - gdas.t{hh}z.atmf{fcst_hr:03d}.nc + nomads: + protocol: download + url: https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/enkfgdas.{yyyymmdd}/{hh}/atmos/mem{mem:03d} + file_names: + netcdf: + fcst: + - gdas.t{hh}z.sfcf{fcst_hr:03d}.nc + aws: + protocol: download + url: https://noaa-gfs-bdp-pds.s3.amazonaws.com/enkfgdas.{yyyymmdd}/{hh}/atmos/mem{mem:03d} + file_names: + netcdf: + anl: + - gdas.t{hh}z.sfcf{fcst_hr:03d}.nc + +GEFS: + aws: + protocol: download + url: + - ['https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}/atmos/pgrb2ap5', 'https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}/atmos/pgrb2bp5'] + - ['https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}/pgrb2a', 'https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}/pgrb2b'] + - ['https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}', 'https://noaa-gefs-pds.s3.amazonaws.com/gefs.{yyyymmdd}/{hh}'] + file_names: + - ['gep{mem:02d}.t{hh}z.pgrb2a.0p50.f{fcst_hr:03d}', 'gep{mem:02d}.t{hh}z.pgrb2b.0p50.f{fcst_hr:03d}'] + - ['gep{mem:02d}.t{hh}z.pgrb2af{fcst_hr:02d}', 'gep{mem:02d}.t{hh}z.pgrb2bf{fcst_hr:02d}'] + - ['gep{mem:02d}.t{hh}z.pgrb2af{fcst_hr:03d}', 'gep{mem:02d}.t{hh}z.pgrb2bf{fcst_hr:03d}' ] + GSMGFS: hpss: protocol: htar @@ -186,7 +248,10 @@ NAM: archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + - ./ archive_file_names: + - com_nam_prod_nam.{yyyymmddhh}.awphys.tar - com_nam_prod_nam.{yyyymmddhh}.awphys{fcst_hr:02d}.tar file_names: anl: diff --git a/ush/test_retrieve_data.py b/ush/test_retrieve_data.py new file mode 100644 index 0000000000..78cc0e8cca --- /dev/null +++ b/ush/test_retrieve_data.py @@ -0,0 +1,319 @@ +''' +Functional test suite for gathering data using retreve_data.py. + +The tests reflect some use cases of gathering various model input data +from HPSS and AWS. Obviously, HPSS tests will only be runnable on +machines with access to NOAA's HPSS system. AWS tests will be runnable +on any platform with an internet connection. + +To run the full test suite: + + python -m unittest -b test_retrieve_data.py + + +To run a single test: + + python -m unittest -b test_retrieve_data.FunctionalTesting.test_rap_lbcs_from_aws +''' +import glob +import os +import tempfile +import unittest + +import retrieve_data + +class FunctionalTesting(unittest.TestCase): + + ''' Test class for retrieve data ''' + + def setUp(self): + self.path = os.path.dirname(__file__) + self.config = f'{self.path}/templates/data_locations.yml' + + @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + def test_fv3gfs_grib2_lbcs_from_hpss(self): + + ''' Get FV3GFS grib2 files from HPSS for LBCS, offset by 6 hours + + ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'fcst', + '--config', self.config, + '--cycle_date', '2022062512', + '--data_stores', 'hpss', + '--external_model', 'FV3GFS', + '--fcst_hrs', '6', '12', '3', + '--output_path', tmp_dir, + '--debug', + '--file_type', 'grib2', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 3) + + @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + def test_fv3gfs_netcdf_lbcs_from_hpss(self): + + ''' Get FV3GFS netcdf files from HPSS for LBCS. Tests fcst lead + times > 40 hours, since they come from a different archive file. + ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'fcst', + '--config', self.config, + '--cycle_date', '2022060112', + '--data_stores', 'hpss', + '--external_model', 'FV3GFS', + '--fcst_hrs', '24', '48', '24', + '--output_path', tmp_dir, + '--debug', + '--file_type', 'netcdf', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 2) + + # GDAS Tests + def test_gdas_ics_from_aws(self): + + ''' In real time, GDAS is used for LBCS with a 6 hour offset. + ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + out_path_tmpl = f'{tmp_dir}/mem{{mem:03d}}' + + args = [ + '--anl_or_fcst', 'anl', + '--config', self.config, + '--cycle_date', '2022052512', + '--data_stores', 'aws', + '--external_model', 'GDAS', + '--fcst_hrs', '6', '9', '3', + '--output_path', out_path_tmpl, + '--debug', + '--file_type', 'netcdf', + '--members', '9', '10', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + for mem in [9, 10]: + files_on_disk = glob.glob( + os.path.join(out_path_tmpl.format(mem=mem), '*') + ) + self.assertEqual(len(files_on_disk), 2) + + + # GEFS Tests + def test_gefs_grib2_ics_from_aws(self): + + ''' Get GEFS grib2 a & b files for ICS offset by 6 hours. + + ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + out_path_tmpl = f'{tmp_dir}/mem{{mem:03d}}' + + args = [ + '--anl_or_fcst', 'anl', + '--config', self.config, + '--cycle_date', '2022052512', + '--data_stores', 'aws', + '--external_model', 'GEFS', + '--fcst_hrs', '6', + '--output_path', out_path_tmpl, + '--debug', + '--file_type', 'netcdf', + '--members', '1', '2', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + for mem in [1, 2]: + files_on_disk = glob.glob( + os.path.join(out_path_tmpl.format(mem=mem), '*') + ) + self.assertEqual(len(files_on_disk), 2) + + + + # HRRR Tests + @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + def test_hrrr_ics_from_hpss(self): + + ''' Get HRRR ICS from hpss ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'anl', + '--config', self.config, + '--cycle_date', '2022062512', + '--data_stores', 'hpss', + '--external_model', 'HRRR', + '--fcst_hrs', '0', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 1) + + @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + def test_hrrr_lbcs_from_hpss(self): + + ''' Get HRRR LBCS from hpss for 3 hour boundary conditions ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'fcst', + '--config', self.config, + '--cycle_date', '2022062512', + '--data_stores', 'hpss', + '--external_model', 'HRRR', + '--fcst_hrs', '3', '24', '3', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 8) + + def test_hrrr_ics_from_aws(self): + + ''' Get HRRR ICS from aws ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'anl', + '--config', self.config, + '--cycle_date', '2022062512', + '--data_stores', 'aws', + '--external_model', 'HRRR', + '--fcst_hrs', '0', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 1) + + def test_hrrr_lbcs_from_aws(self): + + ''' Get HRRR LBCS from aws for 3 hour boundary conditions ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'fcst', + '--config', self.config, + '--cycle_date', '2022062512', + '--data_stores', 'aws', + '--external_model', 'HRRR', + '--fcst_hrs', '3', '24', '3', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 8) + + # RAP tests + def test_rap_ics_from_aws(self): + + ''' Get RAP ICS from aws offset by 3 hours ''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'anl', + '--config', self.config, + '--cycle_date', '2022062509', + '--data_stores', 'aws', + '--external_model', 'RAP', + '--fcst_hrs', '3', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 1) + + def test_rap_lbcs_from_aws(self): + + ''' Get RAP LBCS from aws for 6 hour boundary conditions offset + by 3 hours. Use 09Z start time for longer LBCS.''' + + with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + + args = [ + '--anl_or_fcst', 'fcst', + '--config', self.config, + '--cycle_date', '2022062509', + '--data_stores', 'aws', + '--external_model', 'RAP', + '--fcst_hrs', '3', '30', '6', + '--output_path', tmp_dir, + '--debug', + ] + + retrieve_data.main(args) + + # Verify files exist in temp dir + + os.chdir(os.path.dirname(__file__)) + path = os.path.join(tmp_dir, '*') + files_on_disk = glob.glob(path) + self.assertEqual(len(files_on_disk), 5) From 64157d6aab7bc3fa2b662c54a6ae5a4d563df464 Mon Sep 17 00:00:00 2001 From: Mark Potts <33099090+mark-a-potts@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:51:10 -0400 Subject: [PATCH 198/203] fixed typo (#822) --- scripts/exregional_make_grid.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 929be84dc5..bc3bca1aba 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -319,7 +319,7 @@ Call to python script set_namelist.py to set the variables in the regional_esg_grid namelist file failed. Parameters passed to this script are: Full path to output namelist file: - rgnl_grid_nml_fp = \"${rgn_grid_nml_fp}\" + rgnl_grid_nml_fp = \"${rgnl_grid_nml_fp}\" Namelist settings specified on command line (these have highest precedence): settings = $settings" From a03c194121f5a58b27fd114cdeb1f5779e955144 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Thu, 18 Aug 2022 09:50:54 -0600 Subject: [PATCH 199/203] Bugfix with machine file wrapper. (#823) --- ush/launch_FV3LAM_wflow.sh | 2 +- ush/load_modules_run_task.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index de05d0e087..1f7f692026 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -95,9 +95,9 @@ fi #----------------------------------------------------------------------- # . $exptdir/var_defns.sh +. $USHDIR/source_util_funcs.sh . $USHDIR/source_machine_file.sh . $USHDIR/constants.sh -. $USHDIR/source_util_funcs.sh . $USHDIR/init_env.sh # #----------------------------------------------------------------------- diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 73c7c7eed2..52c25b0b95 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -8,8 +8,8 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_machine_file.sh . $USHDIR/source_util_funcs.sh +. $USHDIR/source_machine_file.sh . $USHDIR/init_env.sh # #----------------------------------------------------------------------- From 00a6559fbd0ca093c18c40eecede6fa96575684d Mon Sep 17 00:00:00 2001 From: michelleharrold Date: Thu, 25 Aug 2022 13:57:46 -0600 Subject: [PATCH 200/203] Mods to METplus vx to address multiple issues (#826) * Mods to METplus conf files: TCDC specifications, correction to level specificatons in point-stat mean and prob files, and added functionality to make METplus output dirs in ex scripts. * Updated comments in MET ex-scripts for creating output directories. * Fixed minor formatting issue in exregional_run_gridstatvx.sh --- scripts/exregional_run_ensgridvx.sh | 20 +++++ scripts/exregional_run_ensgridvx_mean.sh | 9 +++ scripts/exregional_run_ensgridvx_prob.sh | 9 +++ scripts/exregional_run_enspointvx.sh | 10 +++ scripts/exregional_run_enspointvx_mean.sh | 10 +++ scripts/exregional_run_enspointvx_prob.sh | 10 +++ scripts/exregional_run_gridstatvx.sh | 19 +++++ scripts/exregional_run_pointstatvx.sh | 10 +++ .../parm/metplus/EnsembleStat_conus_sfc.conf | 5 +- .../parm/metplus/PointStat_conus_sfc.conf | 3 +- .../metplus/PointStat_conus_sfc_prob.conf | 24 +++--- .../metplus/PointStat_upper_air_mean.conf | 2 +- .../metplus/PointStat_upper_air_prob.conf | 78 +++++++++---------- 13 files changed, 152 insertions(+), 57 deletions(-) diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh index 6d2cc1e403..8b8736ec42 100755 --- a/scripts/exregional_run_ensgridvx.sh +++ b/scripts/exregional_run_ensgridvx.sh @@ -117,6 +117,26 @@ else LOG_SUFFIX=ensgrid_${CDATE}_${VAR} fi +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat" # Output directory for ensemble_stat tool. + +# +# If the variable is accumulated precipitation for a time interval +# (bucket) other than 1 hour, the MET/METplus tools called below will +# include pcp_combine. In that case, create (if necessary) directories +# needed by pcp_combine. +# +if [ "${VAR}" = "APCP" ] && [ "${ACCUM: -1}" != "1" ]; then + mkdir_vrfy -p "${EXPTDIR}/metprd/pcp_combine" # For observations + mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/pcp_combine" # For forecast +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh index 9a18823eff..faa56d5caa 100755 --- a/scripts/exregional_run_ensgridvx_mean.sh +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -115,6 +115,15 @@ else LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR} fi +# +#----------------------------------------------------------------------- +# +# Make sure directory in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_mean" # Output directory for grid_stat tool. + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh index 8f45a53cd9..dc32f599a4 100755 --- a/scripts/exregional_run_ensgridvx_prob.sh +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -115,6 +115,15 @@ else LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR} fi +# +#----------------------------------------------------------------------- +# +# Make sure directory in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_prob" # Output directory for grid_stat tool. + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh index ed1bd35817..3ac4401c27 100755 --- a/scripts/exregional_run_enspointvx.sh +++ b/scripts/exregional_run_enspointvx.sh @@ -107,6 +107,16 @@ export fhr_list # LOG_SUFFIX=enspoint_${CDATE} +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat" # Output directory for ensemble_stat tool. + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh index b9a0bc0c87..cd9ee7004c 100755 --- a/scripts/exregional_run_enspointvx_mean.sh +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -108,6 +108,16 @@ export fhr_list INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat LOG_SUFFIX=enspoint_mean_${CDATE} +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_mean" # Output directory for point_stat tool. + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh index 4d2bf84642..4e025125cb 100755 --- a/scripts/exregional_run_enspointvx_prob.sh +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -108,6 +108,16 @@ export fhr_list INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat LOG_SUFFIX=enspoint_prob_${CDATE} +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. +mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_prob" # Output directory for point_stat tool. + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh index ae7f960fcf..b6b5671814 100755 --- a/scripts/exregional_run_gridstatvx.sh +++ b/scripts/exregional_run_gridstatvx.sh @@ -118,6 +118,25 @@ elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then fi fi +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${OUTPUT_BASE}/metprd/grid_stat" +# +# If the variable is accumulated precipitation for a time interval +# (bucket) other than 1 hour, the MET/METplus tools called below will +# include pcp_combine. In that case, create (if necessary) directories +# needed by pcp_combine. +# +if [ "${VAR}" = "APCP" ] && [ "${ACCUM: -1}" != "1" ]; then + mkdir_vrfy -p "${EXPTDIR}/metprd/pcp_combine" # For observations + mkdir_vrfy -p "${OUTPUT_BASE}/metprd/pcp_combine" # For forecast +fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh index 02a65ed2b9..8afed1dd4a 100755 --- a/scripts/exregional_run_pointstatvx.sh +++ b/scripts/exregional_run_pointstatvx.sh @@ -110,6 +110,16 @@ elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then LOG_SUFFIX=pointstat_${CDATE}_${ENSMEM} fi +# +#----------------------------------------------------------------------- +# +# Make sure directories in which output files will be placed exist. +# +#----------------------------------------------------------------------- +# +mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. +mkdir_vrfy -p "${OUTPUT_BASE}/metprd/point_stat" # Output directory for point_stat tool. + # #----------------------------------------------------------------------- # diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf index d8bb1e54e5..9f93af38f6 100644 --- a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf @@ -215,7 +215,7 @@ ENS_VAR3_OPTIONS = GRIB2_pdt = 0; ;; derive instantaneous 10-m wind from U/V com ENS_VAR4_NAME = TCDC ENS_VAR4_LEVELS = L0 ENS_VAR4_THRESH = <25, >75 -ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = [ 14 ]; GRIB2_ipdtmpl_val = [ 0 ]; interp = { type = [ { method = NEAREST; width = 1; } ]; } +ENS_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } ENS_VAR5_NAME = VIS ENS_VAR5_LEVELS = L0 @@ -261,11 +261,10 @@ OBS_VAR3_OPTIONS = obs_error = { flag = TRUE; dist_type = NONE; dist_parm = []; FCST_VAR4_NAME = TCDC FCST_VAR4_LEVELS = L0 FCST_VAR4_THRESH = <25, >75 -FCST_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index = 8; GRIB2_ipdtmpl_val = {lead?fmt=%H}; +FCST_VAR4_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR4_NAME = TCDC OBS_VAR4_LEVELS = L0 OBS_VAR4_THRESH = <25, >75 -OBS_VAR4_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } BOTH_VAR5_NAME = VIS BOTH_VAR5_LEVELS = L0 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/ush/templates/parm/metplus/PointStat_conus_sfc.conf index 4dfea57023..a0994f222e 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc.conf @@ -212,10 +212,9 @@ BOTH_VAR7_LEVELS = Z0 FCST_VAR8_NAME = TCDC FCST_VAR8_LEVELS = L0 -FCST_VAR8_OPTIONS = GRIB_lvl_typ = 200; interp = { type = [ { method = NEAREST; width = 1; } ]; } +FCST_VAR8_OPTIONS = GRIB_lvl_typ = 200; GRIB2_ipdtmpl_index=[27]; GRIB2_ipdtmpl_val=[255]; interp = { type = [ { method = NEAREST; width = 1; } ]; } OBS_VAR8_NAME = TCDC OBS_VAR8_LEVELS = L0 -OBS_VAR8_OPTIONS = interp = { type = [ { method = NEAREST; width = 1; } ]; } BOTH_VAR9_NAME = VIS BOTH_VAR9_LEVELS = L0 diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf index dc85505ebc..66bbc06020 100644 --- a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf @@ -185,7 +185,7 @@ POINT_STAT_ONCE_PER_FIELD = False # Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set # To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_Z2_ENS_FREQ_ge268 -FCST_VAR1_LEVELS = (*,*) +FCST_VAR1_LEVELS = Z2 FCST_VAR1_THRESH = ==0.1 OBS_VAR1_NAME = TMP @@ -193,7 +193,7 @@ OBS_VAR1_LEVELS = Z2 OBS_VAR1_THRESH = >=268 FCST_VAR2_NAME = TMP_Z2_ENS_FREQ_ge273 -FCST_VAR2_LEVELS = (*,*) +FCST_VAR2_LEVELS = Z2 FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = TMP @@ -201,7 +201,7 @@ OBS_VAR2_LEVELS = Z2 OBS_VAR2_THRESH = >=273 FCST_VAR3_NAME = TMP_Z2_ENS_FREQ_ge278 -FCST_VAR3_LEVELS = (*,*) +FCST_VAR3_LEVELS = Z2 FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = TMP @@ -209,7 +209,7 @@ OBS_VAR3_LEVELS = Z2 OBS_VAR3_THRESH = >=278 FCST_VAR4_NAME = TMP_Z2_ENS_FREQ_ge293 -FCST_VAR4_LEVELS = (*,*) +FCST_VAR4_LEVELS = Z2 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = TMP @@ -217,7 +217,7 @@ OBS_VAR4_LEVELS = Z2 OBS_VAR4_THRESH = >=293 FCST_VAR5_NAME = TMP_Z2_ENS_FREQ_ge298 -FCST_VAR5_LEVELS = (*,*) +FCST_VAR5_LEVELS = Z2 FCST_VAR5_THRESH = ==0.1 OBS_VAR5_NAME = TMP @@ -225,7 +225,7 @@ OBS_VAR5_LEVELS = Z2 OBS_VAR5_THRESH = >=298 FCST_VAR6_NAME = TMP_Z2_ENS_FREQ_ge303 -FCST_VAR6_LEVELS = (*,*) +FCST_VAR6_LEVELS = Z2 FCST_VAR6_THRESH = ==0.1 OBS_VAR6_NAME = TMP @@ -233,7 +233,7 @@ OBS_VAR6_LEVELS = Z2 OBS_VAR6_THRESH = >=303 FCST_VAR7_NAME = DPT_Z2_ENS_FREQ_ge263 -FCST_VAR7_LEVELS = (*,*) +FCST_VAR7_LEVELS = Z2 FCST_VAR7_THRESH = ==0.1 OBS_VAR7_NAME = DPT @@ -241,7 +241,7 @@ OBS_VAR7_LEVELS = Z2 OBS_VAR7_THRESH = >=263 FCST_VAR8_NAME = DPT_Z2_ENS_FREQ_ge268 -FCST_VAR8_LEVELS = (*,*) +FCST_VAR8_LEVELS = Z2 FCST_VAR8_THRESH = ==0.1 OBS_VAR8_NAME = DPT @@ -249,7 +249,7 @@ OBS_VAR8_LEVELS = Z2 OBS_VAR8_THRESH = >=268 FCST_VAR9_NAME = DPT_Z2_ENS_FREQ_ge273 -FCST_VAR9_LEVELS = (*,*) +FCST_VAR9_LEVELS = Z2 FCST_VAR9_THRESH = ==0.1 OBS_VAR9_NAME = DPT @@ -257,7 +257,7 @@ OBS_VAR9_LEVELS = Z2 OBS_VAR9_THRESH = >=273 FCST_VAR10_NAME = DPT_Z2_ENS_FREQ_ge288 -FCST_VAR10_LEVELS = (*,*) +FCST_VAR10_LEVELS = Z2 FCST_VAR10_THRESH = ==0.1 OBS_VAR10_NAME = DPT @@ -265,7 +265,7 @@ OBS_VAR10_LEVELS = Z2 OBS_VAR10_THRESH = >=288 FCST_VAR11_NAME = DPT_Z2_ENS_FREQ_ge293 -FCST_VAR11_LEVELS = (*,*) +FCST_VAR11_LEVELS = Z2 FCST_VAR11_THRESH = ==0.1 OBS_VAR11_NAME = DPT @@ -273,7 +273,7 @@ OBS_VAR11_LEVELS = Z2 OBS_VAR11_THRESH = >=293 FCST_VAR12_NAME = DPT_Z2_ENS_FREQ_ge298 -FCST_VAR12_LEVELS = (*,*) +FCST_VAR12_LEVELS = Z2 FCST_VAR12_THRESH = ==0.1 OBS_VAR12_NAME = DPT diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf index 1acef02f31..4b5e917100 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_mean.conf @@ -216,7 +216,7 @@ OBS_VAR4_NAME = DPT OBS_VAR4_LEVELS = P850 OBS_VAR4_THRESH = >=273, >=278, >=283 -FCST_VAR5_NAME = DPT_P850_ENS_MEAN +FCST_VAR5_NAME = DPT_P700_ENS_MEAN FCST_VAR5_LEVELS = P700 FCST_VAR5_THRESH = >=263, >=286, >=273 diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf index cedb5ec648..213e1f04f5 100644 --- a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf +++ b/ush/templates/parm/metplus/PointStat_upper_air_prob.conf @@ -185,7 +185,7 @@ POINT_STAT_ONCE_PER_FIELD = False # Note: If FCST_VAR_* is set, then a corresponding OBS_VAR_* variable must be set # To use one variables for both forecast and observation data, set BOTH_VAR_* instead FCST_VAR1_NAME = TMP_P850_ENS_FREQ_ge288 -FCST_VAR1_LEVELS = (*,*) +FCST_VAR1_LEVELS = P850 FCST_VAR1_THRESH = ==0.1 OBS_VAR1_NAME = TMP @@ -193,7 +193,7 @@ OBS_VAR1_LEVELS = P850 OBS_VAR1_THRESH = >=288 FCST_VAR2_NAME = TMP_P850_ENS_FREQ_ge293 -FCST_VAR2_LEVELS = (*,*) +FCST_VAR2_LEVELS = P850 FCST_VAR2_THRESH = ==0.1 OBS_VAR2_NAME = TMP @@ -201,7 +201,7 @@ OBS_VAR2_LEVELS = P850 OBS_VAR2_THRESH = >=293 FCST_VAR3_NAME = TMP_P850_ENS_FREQ_ge298 -FCST_VAR3_LEVELS = (*,*) +FCST_VAR3_LEVELS = P850 FCST_VAR3_THRESH = ==0.1 OBS_VAR3_NAME = TMP @@ -209,7 +209,7 @@ OBS_VAR3_LEVELS = P850 OBS_VAR3_THRESH = >=298 FCST_VAR4_NAME = TMP_P700_ENS_FREQ_ge273 -FCST_VAR4_LEVELS = (*,*) +FCST_VAR4_LEVELS = P700 FCST_VAR4_THRESH = ==0.1 OBS_VAR4_NAME = TMP @@ -217,7 +217,7 @@ OBS_VAR4_LEVELS = P700 OBS_VAR4_THRESH = >=273 FCST_VAR5_NAME = TMP_P700_ENS_FREQ_ge278 -FCST_VAR5_LEVELS = (*,*) +FCST_VAR5_LEVELS = P700 FCST_VAR5_THRESH = ==0.1 OBS_VAR5_NAME = TMP @@ -225,7 +225,7 @@ OBS_VAR5_LEVELS = P700 OBS_VAR5_THRESH = >=278 FCST_VAR6_NAME = TMP_P700_ENS_FREQ_ge283 -FCST_VAR6_LEVELS = (*,*) +FCST_VAR6_LEVELS = P700 FCST_VAR6_THRESH = ==0.1 OBS_VAR6_NAME = TMP @@ -233,7 +233,7 @@ OBS_VAR6_LEVELS = P700 OBS_VAR6_THRESH = >=283 FCST_VAR7_NAME = TMP_P500_ENS_FREQ_ge258 -FCST_VAR7_LEVELS = (*,*) +FCST_VAR7_LEVELS = P500 FCST_VAR7_THRESH = ==0.1 OBS_VAR7_NAME = TMP @@ -241,7 +241,7 @@ OBS_VAR7_LEVELS = P500 OBS_VAR7_THRESH = >=258 FCST_VAR8_NAME = TMP_P500_ENS_FREQ_ge263 -FCST_VAR8_LEVELS = (*,*) +FCST_VAR8_LEVELS = P500 FCST_VAR8_THRESH = ==0.1 OBS_VAR8_NAME = TMP @@ -249,7 +249,7 @@ OBS_VAR8_LEVELS = P500 OBS_VAR8_THRESH = >=263 FCST_VAR9_NAME = TMP_P500_ENS_FREQ_ge268 -FCST_VAR9_LEVELS = (*,*) +FCST_VAR9_LEVELS = P500 FCST_VAR9_THRESH = ==0.1 OBS_VAR9_NAME = TMP @@ -257,7 +257,7 @@ OBS_VAR9_LEVELS = P500 OBS_VAR9_THRESH = >=268 FCST_VAR10_NAME = DPT_P850_ENS_FREQ_ge273 -FCST_VAR10_LEVELS = (*,*) +FCST_VAR10_LEVELS = P850 FCST_VAR10_THRESH = ==0.1 OBS_VAR10_NAME = DPT @@ -265,7 +265,7 @@ OBS_VAR10_LEVELS = P850 OBS_VAR10_THRESH = >=273 FCST_VAR11_NAME = DPT_P850_ENS_FREQ_ge278 -FCST_VAR11_LEVELS = (*,*) +FCST_VAR11_LEVELS = P850 FCST_VAR11_THRESH = ==0.1 OBS_VAR11_NAME = DPT @@ -273,7 +273,7 @@ OBS_VAR11_LEVELS = P850 OBS_VAR11_THRESH = >=278 FCST_VAR12_NAME = DPT_P850_ENS_FREQ_ge283 -FCST_VAR12_LEVELS = (*,*) +FCST_VAR12_LEVELS = P850 FCST_VAR12_THRESH = ==0.1 OBS_VAR12_NAME = DPT @@ -281,7 +281,7 @@ OBS_VAR12_LEVELS = P850 OBS_VAR12_THRESH = >=283 FCST_VAR13_NAME = DPT_P700_ENS_FREQ_ge263 -FCST_VAR13_LEVELS = (*,*) +FCST_VAR13_LEVELS = P700 FCST_VAR13_THRESH = ==0.1 OBS_VAR13_NAME = DPT @@ -289,7 +289,7 @@ OBS_VAR13_LEVELS = P700 OBS_VAR13_THRESH = >=263 FCST_VAR14_NAME = DPT_P700_ENS_FREQ_ge268 -FCST_VAR14_LEVELS = (*,*) +FCST_VAR14_LEVELS = P700 FCST_VAR14_THRESH = ==0.1 OBS_VAR14_NAME = DPT @@ -297,7 +297,7 @@ OBS_VAR14_LEVELS = P700 OBS_VAR14_THRESH = >=268 FCST_VAR15_NAME = DPT_P700_ENS_FREQ_ge273 -FCST_VAR15_LEVELS = (*,*) +FCST_VAR15_LEVELS = P700 FCST_VAR15_THRESH = ==0.1 OBS_VAR15_NAME = DPT @@ -305,7 +305,7 @@ OBS_VAR15_LEVELS = P700 OBS_VAR15_THRESH = >=273 FCST_VAR16_NAME = WIND_P850_ENS_FREQ_ge5 -FCST_VAR16_LEVELS = (*,*) +FCST_VAR16_LEVELS = P850 FCST_VAR16_THRESH = ==0.1 OBS_VAR16_NAME = WIND @@ -313,7 +313,7 @@ OBS_VAR16_LEVELS = P850 OBS_VAR16_THRESH = >=5 FCST_VAR17_NAME = WIND_P850_ENS_FREQ_ge10 -FCST_VAR17_LEVELS = (*,*) +FCST_VAR17_LEVELS = P850 FCST_VAR17_THRESH = ==0.1 OBS_VAR17_NAME = WIND @@ -321,7 +321,7 @@ OBS_VAR17_LEVELS = P850 OBS_VAR17_THRESH = >=10 FCST_VAR18_NAME = WIND_P850_ENS_FREQ_ge15 -FCST_VAR18_LEVELS = (*,*) +FCST_VAR18_LEVELS = P850 FCST_VAR18_THRESH = ==0.1 OBS_VAR18_NAME = WIND @@ -329,7 +329,7 @@ OBS_VAR18_LEVELS = P850 OBS_VAR18_THRESH = >=15 FCST_VAR19_NAME = WIND_P700_ENS_FREQ_ge10 -FCST_VAR19_LEVELS = (*,*) +FCST_VAR19_LEVELS = P700 FCST_VAR19_THRESH = ==0.1 OBS_VAR19_NAME = WIND @@ -337,7 +337,7 @@ OBS_VAR19_LEVELS = P700 OBS_VAR19_THRESH = >=10 FCST_VAR20_NAME = WIND_P700_ENS_FREQ_ge15 -FCST_VAR20_LEVELS = (*,*) +FCST_VAR20_LEVELS = P700 FCST_VAR20_THRESH = ==0.1 OBS_VAR20_NAME = WIND @@ -345,7 +345,7 @@ OBS_VAR20_LEVELS = P700 OBS_VAR20_THRESH = >=15 FCST_VAR21_NAME = WIND_P700_ENS_FREQ_ge20 -FCST_VAR21_LEVELS = (*,*) +FCST_VAR21_LEVELS = P700 FCST_VAR21_THRESH = ==0.1 OBS_VAR21_NAME = WIND @@ -353,7 +353,7 @@ OBS_VAR21_LEVELS = P700 OBS_VAR21_THRESH = >=20 FCST_VAR22_NAME = WIND_P500_ENS_FREQ_ge15 -FCST_VAR22_LEVELS = (*,*) +FCST_VAR22_LEVELS = P500 FCST_VAR22_THRESH = ==0.1 OBS_VAR22_NAME = WIND @@ -361,7 +361,7 @@ OBS_VAR22_LEVELS = P500 OBS_VAR22_THRESH = >=15 FCST_VAR23_NAME = WIND_P500_ENS_FREQ_ge21 -FCST_VAR23_LEVELS = (*,*) +FCST_VAR23_LEVELS = P500 FCST_VAR23_THRESH = ==0.1 OBS_VAR23_NAME = WIND @@ -369,7 +369,7 @@ OBS_VAR23_LEVELS = P500 OBS_VAR23_THRESH = >=21 FCST_VAR24_NAME = WIND_P500_ENS_FREQ_ge26 -FCST_VAR24_LEVELS = (*,*) +FCST_VAR24_LEVELS = P500 FCST_VAR24_THRESH = ==0.1 OBS_VAR24_NAME = WIND @@ -377,7 +377,7 @@ OBS_VAR24_LEVELS = P500 OBS_VAR24_THRESH = >=26 FCST_VAR25_NAME = WIND_P250_ENS_FREQ_ge26 -FCST_VAR25_LEVELS = (*,*) +FCST_VAR25_LEVELS = P250 FCST_VAR25_THRESH = ==0.1 OBS_VAR25_NAME = WIND @@ -385,7 +385,7 @@ OBS_VAR25_LEVELS = P250 OBS_VAR25_THRESH = >=26 FCST_VAR26_NAME = WIND_P250_ENS_FREQ_ge31 -FCST_VAR26_LEVELS = (*,*) +FCST_VAR26_LEVELS = P250 FCST_VAR26_THRESH = ==0.1 OBS_VAR26_NAME = WIND @@ -393,7 +393,7 @@ OBS_VAR26_LEVELS = P250 OBS_VAR26_THRESH = >=31 FCST_VAR27_NAME = WIND_P250_ENS_FREQ_ge36 -FCST_VAR27_LEVELS = (*,*) +FCST_VAR27_LEVELS = P250 FCST_VAR27_THRESH = ==0.1 OBS_VAR27_NAME = WIND @@ -401,7 +401,7 @@ OBS_VAR27_LEVELS = P250 OBS_VAR27_THRESH = >=36 FCST_VAR28_NAME = WIND_P250_ENS_FREQ_ge46 -FCST_VAR28_LEVELS = (*,*) +FCST_VAR28_LEVELS = P250 FCST_VAR28_THRESH = ==0.1 OBS_VAR28_NAME = WIND @@ -409,7 +409,7 @@ OBS_VAR28_LEVELS = P250 OBS_VAR28_THRESH = >=46 FCST_VAR29_NAME = WIND_P250_ENS_FREQ_ge62 -FCST_VAR29_LEVELS = (*,*) +FCST_VAR29_LEVELS = P250 FCST_VAR29_THRESH = ==0.1 OBS_VAR29_NAME = WIND @@ -417,7 +417,7 @@ OBS_VAR29_LEVELS = P250 OBS_VAR29_THRESH = >=62 FCST_VAR30_NAME = HGT_P500_ENS_FREQ_ge5400 -FCST_VAR30_LEVELS = (*,*) +FCST_VAR30_LEVELS = P500 FCST_VAR30_THRESH = ==0.1 OBS_VAR30_NAME = HGT @@ -425,7 +425,7 @@ OBS_VAR30_LEVELS = P500 OBS_VAR30_THRESH = >=5400 FCST_VAR31_NAME = HGT_P500_ENS_FREQ_ge5600 -FCST_VAR31_LEVELS = (*,*) +FCST_VAR31_LEVELS = P500 FCST_VAR31_THRESH = ==0.1 OBS_VAR31_NAME = HGT @@ -433,7 +433,7 @@ OBS_VAR31_LEVELS = P500 OBS_VAR31_THRESH = >=5600 FCST_VAR32_NAME = HGT_P500_ENS_FREQ_ge5880 -FCST_VAR32_LEVELS = (*,*) +FCST_VAR32_LEVELS = P500 FCST_VAR32_THRESH = ==0.1 OBS_VAR32_NAME = HGT @@ -441,7 +441,7 @@ OBS_VAR32_LEVELS = P500 OBS_VAR32_THRESH = >=5880 FCST_VAR33_NAME = CAPE_L0_ENS_FREQ_le1000 -FCST_VAR33_LEVELS = (*,*) +FCST_VAR33_LEVELS = L0 FCST_VAR33_THRESH = ==0.1 OBS_VAR33_NAME = CAPE @@ -450,7 +450,7 @@ OBS_VAR33_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; OBS_VAR33_THRESH = <=1000 FCST_VAR34_NAME = CAPE_L0_ENS_FREQ_gt1000.and.lt2500 -FCST_VAR34_LEVELS = (*,*) +FCST_VAR34_LEVELS = L0 FCST_VAR34_THRESH = ==0.1 OBS_VAR34_NAME = CAPE @@ -459,7 +459,7 @@ OBS_VAR34_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; OBS_VAR34_THRESH = >1000&&<2500 FCST_VAR35_NAME = CAPE_L0_ENS_FREQ_gt2500.and.lt4000 -FCST_VAR35_LEVELS = (*,*) +FCST_VAR35_LEVELS = L0 FCST_VAR35_THRESH = ==0.1 OBS_VAR35_NAME = CAPE @@ -468,7 +468,7 @@ OBS_VAR35_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; OBS_VAR35_THRESH = >2500&&<4000 FCST_VAR36_NAME = CAPE_L0_ENS_FREQ_gt2500 -FCST_VAR36_LEVELS = (*,*) +FCST_VAR36_LEVELS = L0 FCST_VAR36_THRESH = ==0.1 OBS_VAR36_NAME = CAPE @@ -477,7 +477,7 @@ OBS_VAR36_OPTIONS = cnt_thresh = [ >0 ]; cnt_logic = UNION; OBS_VAR36_THRESH =>2500 FCST_VAR37_NAME = HPBL_Z0_ENS_FREQ_lt500 -FCST_VAR37_LEVELS = (*,*) +FCST_VAR37_LEVELS = Z0 FCST_VAR37_THRESH = ==0.1 OBS_VAR37_NAME = PBL @@ -486,7 +486,7 @@ OBS_VAR37_OPTIONS = desc = "TKE"; OBS_VAR37_THRESH = <500 FCST_VAR38_NAME = HPBL_Z0_ENS_FREQ_lt1500 -FCST_VAR38_LEVELS = (*,*) +FCST_VAR38_LEVELS = Z0 FCST_VAR38_THRESH = ==0.1 OBS_VAR38_NAME = PBL @@ -495,7 +495,7 @@ OBS_VAR38_OPTIONS = desc = "TKE"; OBS_VAR38_THRESH = <1500 FCST_VAR39_NAME = HPBL_Z0_ENS_FREQ_gt1500 -FCST_VAR39_LEVELS = (*,*) +FCST_VAR39_LEVELS = Z0 FCST_VAR39_THRESH = ==0.1 OBS_VAR39_NAME = PBL From 4a5af68438584ae5923d13501c935f0e63c01e7c Mon Sep 17 00:00:00 2001 From: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Date: Thu, 25 Aug 2022 19:00:22 -0600 Subject: [PATCH 201/203] Use new MET/METplus installs on Gaea (#824) * add paths to recent MET/METplus installations on Gaea * change data staging directory to ncep_shared --- ush/machine/gaea.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh index 88f7153665..036b005e90 100755 --- a/ush/machine/gaea.sh +++ b/ush/machine/gaea.sh @@ -48,7 +48,7 @@ QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} WTIME_MAKE_LBCS="00:60:00" # UFS SRW App specific paths -staged_data_dir="/lustre/f2/pdata/ncep/UFS_SRW_App/develop" +staged_data_dir="/lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop" FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} @@ -66,5 +66,11 @@ RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' # MET Installation Locations -# MET Plus is not yet supported on gaea +MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/usw/met/10.1.2"} +METPLUS_PATH=${METPLUS_PATH:-"/usw/met/METplus/METplus-4.1.3"} +CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} +MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} +NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} +MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} + # Test Data Locations From 67a074a8bcaf0eb65ae30370cfe4390e329bd45e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 31 Aug 2022 21:14:49 -0600 Subject: [PATCH 202/203] Remove regional_workflow from Externals.cfg --- Externals.cfg | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index c595c96b98..db4bfff00a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,12 +1,3 @@ -[regional_workflow] -protocol = git -repo_url = https://github.com/ufs-community/regional_workflow -# Specify either a branch name or a hash but not both. -#branch = develop -hash = 3a2cb82 -local_path = regional_workflow -required = True - [ufs_utils] protocol = git repo_url = https://github.com/ufs-community/UFS_UTILS From 9f6a04b21a8e426be94a73837702e5cbd881a128 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 31 Aug 2022 22:17:40 -0600 Subject: [PATCH 203/203] Modifications to allow WE2E tests to run in new merged structure. - Replace all instances of "homerrfs" with "SR_WX_APP_TOP_DIR", and update location appropriately - Remove references to "regional_workflow" and update paths accordingly --- tests/WE2E/get_expts_status.sh | 10 +++++----- tests/WE2E/run_WE2E_tests.sh | 20 ++++++++++---------- ush/config_defaults.sh | 12 ++++++------ ush/load_modules_run_task.sh | 10 +++++----- ush/setup.py | 26 +++++++------------------- 5 files changed, 33 insertions(+), 45 deletions(-) diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index a17b4e08fa..108ec68e35 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -44,13 +44,13 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # # The current script should be located in the "tests" subdirectory of the -# workflow's top-level directory, which we denote by homerrfs. Thus, -# homerrfs is the directory one level above the directory in which the -# current script is located. Set homerrfs accordingly. +# workflow's top-level directory, which we denote by SR_WX_APP_TOP_DIR. +# SR_WX_APP_TOP_DIR is the directory two levels above the directory in +# which the current script is located; set SR_WX_APP_TOP_DIR accordingly. # #----------------------------------------------------------------------- # -homerrfs=${scrfunc_dir%/*/*} +SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # @@ -58,7 +58,7 @@ homerrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # -ushdir="$homerrfs/ush" +ushdir="$SR_WX_APP_TOP_DIR/ush" # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 605a9178a3..7333c25d36 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -27,24 +27,24 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -# Set the full path to the top-level directory of the regional_workflow -# repository. We denote this path by homerrfs. The current script +# Set the full path to the top-level directory of the UFS SRW App +# repository. We denote this path by SR_WX_APP_TOP_DIR. The current script # should be located in the "tests/WE2E" subdirectory under this directory. -# Thus, homerrfs is the directory two levels above the directory in which +# Thus, SR_WX_APP_TOP_DIR is the directory two levels above the directory in which # the current script is located. # #----------------------------------------------------------------------- # -homerrfs=${scrfunc_dir%/*/*} +SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # -# Set other directories that depend on homerrfs. +# Set other directories that depend on SR_WX_APP_TOP_DIR. # #----------------------------------------------------------------------- # -ushdir="$homerrfs/ush" -testsdir="$homerrfs/tests" +ushdir="$SR_WX_APP_TOP_DIR/ush" +testsdir="$SR_WX_APP_TOP_DIR/tests" WE2Edir="$testsdir/WE2E" # #----------------------------------------------------------------------- @@ -203,9 +203,9 @@ STMP in the configuration files of all tests that will run in NCO mode. If it is not specified, STMP is (effectively) set as follows in the configuration files (of all NCO mode tests to be run): - STMP=\$( readlink -f \"\$homerrfs/../../nco_dirs/stmp\" \) + STMP=\$( readlink -f \"\$SR_WX_APP_TOP_DIR/../nco_dirs/stmp\" \) -Here, homerrfs is the base directory in which the regional_workflow +Here, SR_WX_APP_TOP_DIR is the base directory in which the UFS SRW App repository is cloned. Note that it is not possible to specify a different value for STMP for each test via this argument; all tests will use the same value for STMP (either the value specified in the call to this @@ -993,7 +993,7 @@ COMIN=\"${COMIN}\"" # # Set STMP and PTMP. # - nco_basedir=$( readlink -f "$homerrfs/../../nco_dirs" ) + nco_basedir=$( readlink -f "$SR_WX_APP_TOP_DIR/../nco_dirs" ) STMP=${stmp:-"${nco_basedir}/stmp"} PTMP=${ptmp:-"${nco_basedir}/ptmp"} diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 6c9dc7b032..3eab91ba0d 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -207,7 +207,7 @@ CRON_RELAUNCH_INTVL_MNTS="03" # EXPT_BASEDIR: # The base directory in which the experiment directory will be created. # If this is not specified or if it is set to an empty string, it will -# default to ${HOMErrfs}/../expt_dirs. +# default to ${SR_WX_APP_TOP_DIR}/../expt_dirs. # # EXPT_SUBDIR: # The name that the experiment directory (without the full path) will @@ -519,7 +519,7 @@ WRITE_DOPOST="FALSE" # files under the '00' directory). The script to pull the CCPA data # from the NOAA HPSS has an example of how to account for this as well # as organizing the data into a more intuitive format: -# regional_workflow/scripts/exregional_get_ccpa_files.sh. When a fix +# scripts/exregional_get_ccpa_files.sh. When a fix # is provided, it will be accounted for in the # exregional_get_ccpa_files.sh script. # @@ -548,9 +548,9 @@ WRITE_DOPOST="FALSE" # and rename the MRMS composite reflectivity file to match the valid # time of the forecast. The script to pull the MRMS data from the NOAA # HPSS has an example of the expected file naming structure: -# regional_workflow/scripts/exregional_get_mrms_files.sh. This script +# scripts/exregional_get_mrms_files.sh. This script # calls the script used to identify the MRMS file closest to the valid -# time: regional_workflow/ush/mrms_pull_topofhour.py. +# time: ush/mrms_pull_topofhour.py. # # NDAS_OBS_DIR: # User-specified location of top-level directory where NDAS prepbufr @@ -571,7 +571,7 @@ WRITE_DOPOST="FALSE" # month, and HH the 2-digit valid hour of the day. The script to pull # the NDAS data from the NOAA HPSS has an example of how to rename the # NDAS data into a more intuitive format with the valid time listed in -# the file name: regional_workflow/scripts/exregional_get_ndas_files.sh +# the file name: scripts/exregional_get_ndas_files.sh # #----------------------------------------------------------------------- # @@ -1176,7 +1176,7 @@ WRTCMP_dy="" # commonly used set of grid-dependent parameters. The predefined grid # parameters are specified in the script # -# $HOMErrfs/ush/set_predef_grid_params.py +# $SR_WX_APP_TOP_DIR/ush/set_predef_grid_params.py # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 52c25b0b95..385ff824b2 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -111,23 +111,23 @@ for the workflow task specified by task_name failed: # sets environment variables (including prepending/appending to paths) # and loads modules. # -# The regional_workflow repository contains module files for the +# The UFS SRW App repository contains module files for the # workflow tasks in the template rocoto XML file for the FV3-LAM work- # flow that need modules not loaded in the BUILD_MOD_FN above. # # The full path to a module file for a given task is # -# $HOMErrfs/modulefiles/$machine/${task_name}.local +# $SR_WX_APP_TOP_DIR/modulefiles/$machine/${task_name}.local # -# where HOMErrfs is the base directory of the workflow, machine is the +# where SR_WX_APP_TOP_DIR is the base directory of the App, machine is the # name of the machine that we're running on (in lowercase), and task_- # name is the name of the current task (an input to this script). # #----------------------------------------------------------------------- # -modules_dir="$HOMErrfs/modulefiles/tasks/$machine" +modules_dir="$SR_WX_APP_TOP_DIR/modulefiles/tasks/$machine" modulefile_name="${task_name}" -default_modules_dir="$HOMErrfs/modulefiles" +default_modules_dir="$SR_WX_APP_TOP_DIR/modulefiles" # #----------------------------------------------------------------------- # diff --git a/ush/setup.py b/ush/setup.py index 6a8f35231d..570dc674ba 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -229,7 +229,7 @@ def setup(): # directory of the current script. # SR_WX_APP_TOP_DIR=os.path.abspath( os.path.dirname(__file__) + \ - os.sep + os.pardir + os.sep + os.pardir) + os.sep + os.pardir) # #----------------------------------------------------------------------- @@ -252,17 +252,6 @@ def setup(): property_name="local_path" cfg = load_ini_config(mng_extrns_cfg_fn) # - # Get the path to the workflow scripts - # - external_name="regional_workflow" - HOMErrfs = get_ini_value(cfg, external_name, property_name) - - if not HOMErrfs: - print_err_msg_exit(f''' - Externals.cfg does not contain "{external_name}".''') - - HOMErrfs = os.path.join(SR_WX_APP_TOP_DIR, HOMErrfs) - # # Get the base directory of the FV3 forecast model code. # external_name=FCST_MODEL @@ -322,13 +311,13 @@ def setup(): global USHDIR, SCRIPTSDIR, JOBSDIR,SORCDIR, SRC_DIR, PARMDIR, MODULES_DIR, EXECDIR, TEMPLATE_DIR, \ VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG - USHDIR = os.path.join(HOMErrfs,"ush") - SCRIPTSDIR = os.path.join(HOMErrfs,"scripts") - JOBSDIR = os.path.join(HOMErrfs,"jobs") - SORCDIR = os.path.join(HOMErrfs,"sorc") + USHDIR = os.path.join(SR_WX_APP_TOP_DIR,"ush") + SCRIPTSDIR = os.path.join(SR_WX_APP_TOP_DIR,"scripts") + JOBSDIR = os.path.join(SR_WX_APP_TOP_DIR,"jobs") + SORCDIR = os.path.join(SR_WX_APP_TOP_DIR,"sorc") SRC_DIR = os.path.join(SR_WX_APP_TOP_DIR,"src") - PARMDIR = os.path.join(HOMErrfs,"parm") - MODULES_DIR = os.path.join(HOMErrfs,"modulefiles") + PARMDIR = os.path.join(SR_WX_APP_TOP_DIR,"parm") + MODULES_DIR = os.path.join(SR_WX_APP_TOP_DIR,"modulefiles") EXECDIR = os.path.join(SR_WX_APP_TOP_DIR,EXEC_SUBDIR) TEMPLATE_DIR = os.path.join(USHDIR,"templates") VX_CONFIG_DIR = os.path.join(TEMPLATE_DIR,"parm") @@ -1836,7 +1825,6 @@ def setup(): #----------------------------------------------------------------------- # 'SR_WX_APP_TOP_DIR': SR_WX_APP_TOP_DIR, - 'HOMErrfs': HOMErrfs, 'USHDIR': USHDIR, 'SCRIPTSDIR': SCRIPTSDIR, 'JOBSDIR': JOBSDIR,